mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-08-09 08:27:00 -07:00
feat(caretaker-evals): add local golden issue collection and firestore sync tools (#28532)
This commit is contained in:
Chad
@@ -0,0 +1,97 @@
|
||||
# Golden Workable Spec Generator System Instructions
|
||||
|
||||
You are an expert software engineering spec synthesizer assistant. Your
|
||||
objective is to analyze a completed GitHub Issue and its associated PR diff,
|
||||
inspect the PR changes, and synthesize a 100% FAIR, high-precision Golden
|
||||
Workable Spec JSON and its evaluation rationale.
|
||||
|
||||
## REQUIRED REASONING WORKFLOW (CHAIN OF THOUGHT)
|
||||
|
||||
Before producing the final JSON object, you MUST execute this 2-Phase reasoning
|
||||
process:
|
||||
|
||||
### Phase 1: PR File & Fix Analysis
|
||||
|
||||
Examine the PR title, PR body, and code diff. Identify all files modified in the
|
||||
PR diff and the changes made in each.
|
||||
|
||||
### Phase 2: The Fairness Pruning Pass (CRITICAL FOR BENCHMARK FAIRNESS)
|
||||
|
||||
For EACH file modified in the PR diff, cross-reference it against the original
|
||||
Issue Description and ask:
|
||||
|
||||
1. _"Was this file strictly required to resolve the user's reported symptom in
|
||||
the issue text?"_
|
||||
2. _"Or is this file a secondary refactoring, un-reported feature extension, or
|
||||
internal architecture cleanup added opportunistically by the PR author?"_
|
||||
|
||||
**STRICT PRUNING RULE:** You MUST PRUNE all secondary refactoring files from
|
||||
`files_to_modify`. Keep ONLY the primary target source file(s) directly
|
||||
responsible for resolving the reported bug.
|
||||
|
||||
## Workable Spec Synthesis Rules
|
||||
|
||||
1. **Golden Spec Rationale (`golden_spec_rationale`):** Focus STRICTLY on what
|
||||
source files were NOT kept (PRUNED) from `files_to_modify` and WHY:
|
||||
- If files modified in the PR diff were pruned (e.g., secondary refactorings,
|
||||
un-reported feature extensions, or internal architecture cleanups),
|
||||
explicitly name each pruned file and explain why it was excluded for
|
||||
benchmark fairness.
|
||||
- If NO source files were pruned, state: _"No source files were pruned; all
|
||||
PR modifications directly address the reported issue."_
|
||||
- Do NOT state obvious rules (such as _"test files were excluded from
|
||||
files_to_modify"_). Keep the rationale focused purely on non-obvious
|
||||
pruning decisions.
|
||||
2. **Source Files Only:** `files_to_modify` inside `workable_spec` MUST contain
|
||||
ONLY primary source code files. Strictly EXCLUDE test files (`*.test.ts`),
|
||||
lockfiles (`package-lock.json`, `yarn.lock`), documentation markdown files,
|
||||
and version bump files. Test files belong ONLY in
|
||||
`testing_strategy.test_file`.
|
||||
3. **Test File Grounding:**
|
||||
- If the PR diff modified or created an automated test file, set
|
||||
`testing_strategy.test_file` to that exact path.
|
||||
- If the PR diff did NOT touch any automated test file, set
|
||||
`testing_strategy.test_file` strictly to `"N/A"`.
|
||||
4. **Concrete Names (If Applicable):** `summary.root_cause` and
|
||||
`implementation_plan.steps` MUST reference specific function names, regular
|
||||
expressions, constants, or data structures modified to fix the reported
|
||||
issue.
|
||||
5. **No Hand-Waving:** Avoid vague, generic, or hand-wavy phrasing (such as
|
||||
_"update the code as needed"_, _"fix the logic"_, or _"adjust accordingly"_).
|
||||
Every step must give concrete, unambiguous technical guidance.
|
||||
|
||||
## Output JSON Template Requirements
|
||||
|
||||
Your final response MUST be a raw JSON object strictly matching this structure.
|
||||
Do not wrap in markdown code blocks:
|
||||
|
||||
```json
|
||||
{
|
||||
"golden_spec_rationale": "Focus strictly on what source files were PRUNED and why (or state 'No source files were pruned; all PR modifications directly address the reported issue').",
|
||||
"workable_spec": {
|
||||
"issue_id": "{owner}/{repo}#{issue_number}",
|
||||
"summary": {
|
||||
"problem": "Concise statement of reported problem strictly matching the issue description.",
|
||||
"root_cause": "Analysis of root cause referencing specific functions/regexes modified in the PR diff if applicable.",
|
||||
"context": "Additional technical context from issue and PR."
|
||||
},
|
||||
"implementation_plan": {
|
||||
"files_to_modify": ["path/to/primary_source_file.ts"],
|
||||
"steps": [
|
||||
"Ordered step-by-step instructions strictly required to implement the fix for reported issue."
|
||||
]
|
||||
},
|
||||
"testing_strategy": {
|
||||
"test_file": "path/to/test_file.test.ts",
|
||||
"expected_behavior": "Description of expected behavior after fix.",
|
||||
"verification_steps": [
|
||||
"Specific test assertions to add/modify or manual CLI verification steps."
|
||||
],
|
||||
"framework": "Testing framework used (e.g. Vitest or 'N/A' if no automated test file is present)."
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Do not include metadata like spam assessment or effort tags. Keep it focused
|
||||
entirely on instructions for code generation and testing.
|
||||
@@ -0,0 +1,125 @@
|
||||
"""
|
||||
Golden Workable Spec Generator Module.
|
||||
|
||||
Uses the Antigravity SDK (google.antigravity) to synthesize a clean, high-precision
|
||||
Workable Spec JSON directly from Issue and PR Diff text using generate_golden_spec.md.
|
||||
"""
|
||||
|
||||
import re
|
||||
import os
|
||||
import json
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
import sys
|
||||
|
||||
# Ensure cloudrun/triage-worker is in sys.path for worker utility imports
|
||||
CARETAKER_DIR = Path(__file__).resolve().parents[3]
|
||||
TRIAGE_WORKER_DIR = CARETAKER_DIR / "cloudrun" / "triage-worker"
|
||||
if str(TRIAGE_WORKER_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TRIAGE_WORKER_DIR))
|
||||
|
||||
load_dotenv()
|
||||
|
||||
from utils.validator import validate_triage_result
|
||||
from utils.agent_logger import extract_final_output
|
||||
from google.antigravity import Agent, LocalAgentConfig
|
||||
from google.antigravity.hooks.policy import deny
|
||||
|
||||
PROMPT_FILE = Path(__file__).parent / "generate_golden_spec.md"
|
||||
|
||||
|
||||
def _parse_llm_json(raw_text: str) -> dict:
|
||||
"""Strips markdown fences and parses LLM JSON with fallback unescaping."""
|
||||
clean = raw_text.strip()
|
||||
if clean.startswith("```"):
|
||||
clean = clean.split("\n", 1)[-1].rsplit("\n", 1)[0].strip()
|
||||
try:
|
||||
data = json.loads(clean, strict=False)
|
||||
except Exception:
|
||||
cleaned = re.sub(r'\\(?![/"bfnrtu]|u[0-9a-fA-F]{4})', r'\\\\', re.sub(r"(?<!\\)\\'", "'", clean))
|
||||
data = json.loads(cleaned, strict=False)
|
||||
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"Expected JSON object from LLM, but got {type(data).__name__}. Raw output:\n{raw_text}")
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def _load_system_instruction() -> str:
|
||||
"""Loads prompt instructions from generate_golden_spec.md."""
|
||||
if not PROMPT_FILE.exists():
|
||||
raise FileNotFoundError(f"Required prompt file missing at: {PROMPT_FILE}")
|
||||
with open(PROMPT_FILE, "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def generate_golden_spec(owner: str, repo: str, issue_number: int, issue_data: dict, pr_data: dict) -> dict:
|
||||
"""
|
||||
Invokes the Antigravity SDK (google.antigravity) Agent using generate_golden_spec.md
|
||||
instructions to synthesize a clean, high-precision Workable Spec JSON and its rationale.
|
||||
Returns a dict with keys: 'workable_spec' and 'golden_spec_rationale'.
|
||||
"""
|
||||
system_instruction = _load_system_instruction()
|
||||
|
||||
# Filter out lockfiles and non-code noise from diff preview
|
||||
raw_diff = pr_data.get("diff", "")
|
||||
filtered_diff_lines = []
|
||||
skip_file = False
|
||||
for line in raw_diff.split("\n"):
|
||||
if line.startswith("diff --git"):
|
||||
if any(x in line for x in ["package-lock.json", "yarn.lock", "pnpm-lock.yaml"]):
|
||||
skip_file = True
|
||||
else:
|
||||
skip_file = False
|
||||
if not skip_file:
|
||||
filtered_diff_lines.append(line)
|
||||
|
||||
filtered_diff = "\n".join(filtered_diff_lines)
|
||||
|
||||
prompt = f"""Target Issue & PR Data for {owner}/{repo}#{issue_number}:
|
||||
|
||||
Issue #{issue_number} Title: {issue_data.get('title', '')}
|
||||
Issue Description / Body:
|
||||
{issue_data.get('body', '')}
|
||||
|
||||
PR #{pr_data.get('number', '')} Title: {pr_data.get('title', '')}
|
||||
PR Body:
|
||||
{pr_data.get('body', '')}
|
||||
|
||||
PR Filtered Code Diff:
|
||||
{filtered_diff}"""
|
||||
|
||||
policies = [deny("*")]
|
||||
|
||||
async def run_spec_agent():
|
||||
config = LocalAgentConfig(
|
||||
system_instructions=system_instruction,
|
||||
api_key=os.environ.get("GEMINI_API_KEY"),
|
||||
policies=policies,
|
||||
)
|
||||
|
||||
print(f"[EVAL] Initializing Antigravity Spec Generator Agent for Issue #{issue_number}...")
|
||||
async with Agent(config) as agent:
|
||||
response = await agent.chat(prompt)
|
||||
resolved_chunks = await response.resolve()
|
||||
raw_text = extract_final_output(resolved_chunks).strip()
|
||||
|
||||
data = _parse_llm_json(raw_text)
|
||||
|
||||
golden_spec_rationale = data.get("golden_spec_rationale", "")
|
||||
workable_spec = data.get("workable_spec", data)
|
||||
|
||||
payload_to_validate = {
|
||||
"triage_metadata": {"quality": "OK", "effort_estimate": "SMALL"},
|
||||
"workable_spec": workable_spec
|
||||
}
|
||||
validate_triage_result(payload_to_validate)
|
||||
print("Schema validation successful!")
|
||||
|
||||
return {
|
||||
"workable_spec": workable_spec,
|
||||
"golden_spec_rationale": golden_spec_rationale
|
||||
}
|
||||
|
||||
return asyncio.run(run_spec_agent())
|
||||
@@ -0,0 +1,103 @@
|
||||
"""
|
||||
GitHub Information & Target Commit SHA Resolution Utility.
|
||||
|
||||
Provides helper functions for querying GitHub REST API, extracting issue/PR metadata,
|
||||
resolving target repository commit SHAs, and assembling golden issue JSON templates.
|
||||
"""
|
||||
|
||||
import os
|
||||
import requests
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
|
||||
def _get_github_headers() -> Dict[str, str]:
|
||||
"""
|
||||
Optionally retrieves GITHUB_TOKEN (or GH_TOKEN) to authenticate requests.
|
||||
"""
|
||||
token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
|
||||
headers = {"Accept": "application/vnd.github.v3+json"}
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
def get_issue_details(owner: str, repo: str, issue_number: int) -> Dict[str, Any]:
|
||||
"""Queries GitHub REST API for issue details (title, body, createdAt, labels)."""
|
||||
url = f"https://api.github.com/repos/{owner}/{repo}/issues/{issue_number}"
|
||||
resp = requests.get(url, headers=_get_github_headers(), timeout=15)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(f"Failed to fetch issue #{issue_number} from GitHub API ({resp.status_code}): {resp.text}")
|
||||
|
||||
data = resp.json()
|
||||
return {
|
||||
"owner": owner,
|
||||
"repo": repo,
|
||||
"number": data.get("number"),
|
||||
"title": data.get("title", ""),
|
||||
"body": data.get("body", "") or "",
|
||||
"createdAt": data.get("created_at", ""),
|
||||
"labels": data.get("labels", [])
|
||||
}
|
||||
|
||||
|
||||
def get_pr_details(owner: str, repo: str, pr_number: int) -> Dict[str, Any]:
|
||||
"""Queries GitHub REST API for PR details (title, body, baseRefOid, patch/diff)."""
|
||||
url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}"
|
||||
headers = _get_github_headers()
|
||||
resp = requests.get(url, headers=headers, timeout=15)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(f"Failed to fetch PR #{pr_number} from GitHub API ({resp.status_code}): {resp.text}")
|
||||
|
||||
data = resp.json()
|
||||
|
||||
# Fetch unified patch/diff
|
||||
diff_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}"
|
||||
diff_headers = headers.copy()
|
||||
diff_headers["Accept"] = "application/vnd.github.v3.diff"
|
||||
diff_resp = requests.get(diff_url, headers=diff_headers, timeout=15)
|
||||
diff_content = diff_resp.text if diff_resp.status_code == 200 else ""
|
||||
|
||||
return {
|
||||
"number": data.get("number"),
|
||||
"title": data.get("title", ""),
|
||||
"body": data.get("body", "") or "",
|
||||
"baseRefOid": data.get("base", {}).get("sha", ""),
|
||||
"diff": diff_content
|
||||
}
|
||||
|
||||
|
||||
def _get_commit_sha_at_timestamp(owner: str, repo: str, created_at: str) -> str:
|
||||
"""Queries GitHub REST API to find the closest commit SHA at or before the given timestamp."""
|
||||
if not created_at:
|
||||
return ""
|
||||
url = f"https://api.github.com/repos/{owner}/{repo}/commits?until={created_at}&per_page=1"
|
||||
resp = requests.get(url, headers=_get_github_headers(), timeout=15)
|
||||
if resp.status_code == 200:
|
||||
commits = resp.json()
|
||||
if isinstance(commits, list) and len(commits) > 0:
|
||||
return commits[0].get("sha", "")
|
||||
return ""
|
||||
|
||||
|
||||
def resolve_target_version(owner: str, repo: str, issue_data: Dict[str, Any], pr_data: Optional[Dict[str, Any]] = None) -> str:
|
||||
"""
|
||||
Resolves the target Git commit SHA for an issue:
|
||||
1. If PR data contains baseRefOid (base commit before PR fix was merged), use that.
|
||||
2. Otherwise, query GitHub REST API for the commit SHA at issue createdAt timestamp via get_commit_sha_at_timestamp().
|
||||
3. Fallback to 'main'.
|
||||
"""
|
||||
if pr_data and pr_data.get("baseRefOid"):
|
||||
return pr_data["baseRefOid"]
|
||||
|
||||
created_at = issue_data.get("createdAt", "")
|
||||
if created_at:
|
||||
try:
|
||||
sha = _get_commit_sha_at_timestamp(owner, repo, created_at)
|
||||
if sha:
|
||||
return sha
|
||||
except Exception as e:
|
||||
print(f"[FETCH_GITHUB] Warning: Could not resolve commit SHA at timestamp: {e}")
|
||||
|
||||
return "main"
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Maintainer CLI tools for dataset management and metrics."""
|
||||
@@ -0,0 +1,113 @@
|
||||
"""
|
||||
Golden Dataset Quality & Effort Metrics Diagnostic CLI Tool.
|
||||
|
||||
CLI Usage:
|
||||
python3 -m evals.triage.tools.dataset_metrics
|
||||
"""
|
||||
|
||||
from collections import Counter
|
||||
from evals.triage.helpers.dataset import load_issues
|
||||
|
||||
VALID_QUALITIES = ["OK", "SPAM", "EMPTY", "NEEDS_INFO", "FEATURE"]
|
||||
VALID_EFFORTS = ["SMALL", "MEDIUM", "LARGE"]
|
||||
|
||||
|
||||
def _validate_spec_integrity(issues) -> bool:
|
||||
"""
|
||||
Validation helper that enforces spec & metadata integrity across the dataset:
|
||||
- Quality MUST be one of: OK, SPAM, EMPTY, NEEDS_INFO, FEATURE.
|
||||
- OK issues MUST have a valid workable spec and effort estimate (SMALL, MEDIUM, LARGE).
|
||||
- Non-OK issues MUST NOT have a workable spec and MUST have an empty effort string ("").
|
||||
Prints ONLY the specific issues causing errors (if any).
|
||||
"""
|
||||
errors = []
|
||||
for data in issues:
|
||||
issue_num = data.get("issue_number", 0)
|
||||
quality = data.get("expected_quality", "")
|
||||
effort = data.get("expected_effort", "")
|
||||
spec = data.get("expected_workable_spec", {})
|
||||
has_spec = bool(spec and isinstance(spec, dict) and len(spec) > 0)
|
||||
|
||||
# 1. Quality validity check
|
||||
if quality not in VALID_QUALITIES:
|
||||
errors.append(f" ❌ Issue #{issue_num}: Quality '{quality}' is invalid! Must be one of: {VALID_QUALITIES}")
|
||||
|
||||
# 2. Spec & Effort checks
|
||||
if quality == "OK":
|
||||
if not has_spec:
|
||||
errors.append(f" ❌ Issue #{issue_num}: Quality is 'OK' but missing workable spec!")
|
||||
elif not (isinstance(spec, dict) and spec.get("summary") and spec.get("implementation_plan")):
|
||||
errors.append(f" ❌ Issue #{issue_num}: Quality is 'OK' but workable spec structure is incomplete!")
|
||||
|
||||
if effort not in VALID_EFFORTS:
|
||||
errors.append(f" ❌ Issue #{issue_num}: Quality is 'OK' but effort '{effort}' is invalid! Must be one of: {VALID_EFFORTS}")
|
||||
else:
|
||||
if has_spec:
|
||||
errors.append(f" ❌ Issue #{issue_num}: Quality is '{quality}' but has unexpected workable spec content: {spec}")
|
||||
if effort != "":
|
||||
errors.append(f" ❌ Issue #{issue_num}: Quality is '{quality}' but has non-empty effort estimate ('{effort}')!")
|
||||
|
||||
if errors:
|
||||
print("\n--- ⚠️ Spec & Metadata Validation Errors ---")
|
||||
for err in errors:
|
||||
print(err)
|
||||
return False
|
||||
else:
|
||||
print("\n ✅ Spec & Metadata Integrity Check: All issues correctly configured.")
|
||||
return True
|
||||
|
||||
|
||||
def compute_metrics() -> bool:
|
||||
issues = load_issues()
|
||||
total_issues = len(issues)
|
||||
|
||||
if total_issues == 0:
|
||||
print("[METRICS] No golden issues found in Firestore.")
|
||||
return True
|
||||
|
||||
qualities = Counter()
|
||||
ok_efforts = Counter()
|
||||
|
||||
for data in issues:
|
||||
quality = data.get("expected_quality", "")
|
||||
effort = data.get("expected_effort", "")
|
||||
|
||||
qualities[quality] += 1
|
||||
if quality == "OK":
|
||||
ok_efforts[effort] += 1
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print(" 📊 GOLDEN DATASET DIAGNOSTIC REPORT (Firestore)")
|
||||
print("=" * 70)
|
||||
print(f"📦 Total Golden Issues: {total_issues}")
|
||||
|
||||
print("\n--- 🏷️ Expected Quality Breakdown ---")
|
||||
for q in VALID_QUALITIES:
|
||||
count = qualities.get(q, 0)
|
||||
pct = (count / total_issues * 100) if total_issues else 0
|
||||
bar = "█" * count
|
||||
print(f" {q:<12}: {count:>2} ({pct:>5.1f}%) {bar}")
|
||||
|
||||
ok_count = qualities.get("OK", 0)
|
||||
print(f"\n--- ⚡ Expected Effort Breakdown (For {ok_count} OK Issues) ---")
|
||||
for e in VALID_EFFORTS:
|
||||
count = ok_efforts.get(e, 0)
|
||||
pct = (count / ok_count * 100) if ok_count else 0
|
||||
bar = "█" * count
|
||||
print(f" {e:<12}: {count:>2} ({pct:>5.1f}%) {bar}")
|
||||
|
||||
# Run clean Spec & Metadata Integrity Check
|
||||
success = _validate_spec_integrity(issues)
|
||||
|
||||
print("=" * 70 + "\n")
|
||||
return success
|
||||
|
||||
|
||||
def main():
|
||||
import sys
|
||||
if not compute_metrics():
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,88 @@
|
||||
"""
|
||||
Golden Issue Generator CLI Tool (Main Entrypoint).
|
||||
|
||||
CLI usage:
|
||||
python3 -m evals.triage.tools.generate_golden_issue --issue <number> [--pr <number>]
|
||||
"""
|
||||
|
||||
import json
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from evals.triage.helpers.github_api import (
|
||||
get_issue_details,
|
||||
get_pr_details,
|
||||
resolve_target_version
|
||||
)
|
||||
from evals.triage.helpers.generate_golden_spec import generate_golden_spec
|
||||
|
||||
OUTPUT_DIR = Path(__file__).parent.parent / "dataset" / "golden-issues"
|
||||
|
||||
|
||||
def generate_golden_issue(owner: str, repo: str, issue_number: int, pr_number: int = None):
|
||||
"""Main orchestrator for generating a brand-new Golden Issue JSON file."""
|
||||
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
file_path = OUTPUT_DIR / f"gemini_cli_{issue_number}.json"
|
||||
|
||||
print(f"Fetching Issue #{issue_number} details from {owner}/{repo}...")
|
||||
issue_data = get_issue_details(owner, repo, issue_number)
|
||||
|
||||
pr_data = {}
|
||||
if pr_number:
|
||||
print(f"Fetching PR #{pr_number} details from {owner}/{repo}...")
|
||||
pr_data = get_pr_details(owner, repo, pr_number)
|
||||
|
||||
workable_spec = {}
|
||||
golden_spec_rationale = ""
|
||||
|
||||
if pr_number:
|
||||
print(f"[EVAL] Generating Golden Workable Spec for Issue #{issue_number} using PR #{pr_number}...")
|
||||
spec_res = generate_golden_spec(owner, repo, issue_number, issue_data, pr_data)
|
||||
workable_spec = spec_res["workable_spec"]
|
||||
golden_spec_rationale = spec_res["golden_spec_rationale"]
|
||||
|
||||
# Extract effort from labels if present
|
||||
labels = [l.get("name", "").lower() for l in issue_data.get("labels", []) if isinstance(l, dict)]
|
||||
effort_from_labels = ""
|
||||
for effort in ["small", "medium", "large"]:
|
||||
if f"effort/{effort}" in labels:
|
||||
effort_from_labels = effort.upper()
|
||||
break
|
||||
|
||||
# Default quality to 'OK' if a PR is attached, otherwise empty string ''
|
||||
expected_quality_default = "OK" if pr_number else ""
|
||||
|
||||
template = {
|
||||
"owner": owner,
|
||||
"repo": repo,
|
||||
"issue_number": issue_number,
|
||||
"issue_title": issue_data.get("title", ""),
|
||||
"issue_body": issue_data.get("body", ""),
|
||||
"pr_number": pr_number or 0,
|
||||
"target_version": resolve_target_version(owner, repo, issue_data, pr_data),
|
||||
"expected_quality": expected_quality_default,
|
||||
"expected_effort": effort_from_labels,
|
||||
"notes": f"Created at {issue_data.get('createdAt', '')} by automated generate_golden_issue.py",
|
||||
"golden_spec_rationale": golden_spec_rationale,
|
||||
"expected_workable_spec": workable_spec
|
||||
}
|
||||
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
json.dump(template, f, indent=2)
|
||||
|
||||
print(f"Successfully saved golden issue file to: {file_path}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Generate a Golden Issue JSON file.")
|
||||
parser.add_argument("--issue", type=int, required=True, help="GitHub Issue number")
|
||||
parser.add_argument("--pr", type=int, default=None, help="Associated PR number (optional)")
|
||||
parser.add_argument("--owner", type=str, default="google-gemini", help="Repository owner")
|
||||
parser.add_argument("--repo", type=str, default="gemini-cli", help="Repository name")
|
||||
|
||||
args = parser.parse_args()
|
||||
generate_golden_issue(args.owner, args.repo, args.issue, args.pr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,83 @@
|
||||
"""
|
||||
Bidirectional Firestore Synchronization CLI Tool.
|
||||
|
||||
CLI Usage:
|
||||
python3 -m evals.triage.tools.sync_firestore --to-firestore
|
||||
python3 -m evals.triage.tools.sync_firestore --from-firestore
|
||||
"""
|
||||
|
||||
import json
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
from google.cloud import firestore
|
||||
from evals.triage.helpers.dataset import get_env_var
|
||||
|
||||
load_dotenv()
|
||||
|
||||
TRIAGE_EVAL_DIR = Path(__file__).resolve().parent.parent
|
||||
GOLDEN_ISSUES_DIR = TRIAGE_EVAL_DIR / "dataset" / "golden-issues"
|
||||
|
||||
|
||||
def _get_db():
|
||||
project_id = get_env_var("PROJECT_ID")
|
||||
db_id = get_env_var("FIRESTORE_DATABASE")
|
||||
collection_name = get_env_var("FIRESTORE_EVAL_COLLECTION")
|
||||
db = firestore.Client(project=project_id, database=db_id)
|
||||
return db, collection_name
|
||||
|
||||
|
||||
def sync_to_firestore():
|
||||
db, collection_name = _get_db()
|
||||
json_files = sorted([f for f in GOLDEN_ISSUES_DIR.glob("**/gemini_cli_*.json") if not f.name.startswith(".")])
|
||||
if not json_files:
|
||||
print(f"[SYNC] No JSON files found in {GOLDEN_ISSUES_DIR}.")
|
||||
return
|
||||
|
||||
print(f"[SYNC] Uploading {len(json_files)} JSON file(s) to Firestore collection '{collection_name}'...")
|
||||
for file_path in json_files:
|
||||
filename = file_path.name
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
doc_id = f"github_{data['owner']}_{data['repo']}_{data['issue_number']}"
|
||||
db.collection(collection_name).document(doc_id).set(data)
|
||||
print(f" -> Uploaded '{filename}' as '{doc_id}'")
|
||||
except Exception as e:
|
||||
print(f" -> Failed to upload '{filename}': {e}")
|
||||
|
||||
|
||||
def sync_from_firestore():
|
||||
db, collection_name = _get_db()
|
||||
GOLDEN_ISSUES_DIR.mkdir(parents=True, exist_ok=True)
|
||||
docs = db.collection(collection_name).stream()
|
||||
|
||||
count = 0
|
||||
print(f"[SYNC] Downloading documents from Firestore collection '{collection_name}'...")
|
||||
for doc in docs:
|
||||
data = doc.to_dict()
|
||||
issue_num = data.get("issue_number")
|
||||
if not issue_num:
|
||||
continue
|
||||
file_path = GOLDEN_ISSUES_DIR / f"gemini_cli_{int(issue_num)}.json"
|
||||
file_path.write_text(json.dumps(data, indent=2), encoding="utf-8")
|
||||
print(f" -> Downloaded Issue #{issue_num} to '{file_path.name}'")
|
||||
count += 1
|
||||
print(f"[SYNC] Downloaded {count} file(s) to {GOLDEN_ISSUES_DIR}.")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Bidirectional Firestore Synchronization CLI Tool.")
|
||||
group = parser.add_mutually_exclusive_group()
|
||||
group.add_argument("--to-firestore", action="store_true", help="Upload local JSONs to Firestore (Default)")
|
||||
group.add_argument("--from-firestore", action="store_true", help="Download Firestore docs to local JSONs")
|
||||
|
||||
args = parser.parse_args()
|
||||
if args.from_firestore:
|
||||
sync_from_firestore()
|
||||
else:
|
||||
sync_to_firestore()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user