From cd5ac173cff11ef233aaf72d2d96dbb77accef69 Mon Sep 17 00:00:00 2001
From: Chad <chaddiao@google.com>
Date: Fri, 7 Aug 2026 14:46:30 -0500
Subject: [PATCH] feat(caretaker-evals): add Cloud Run job entrypoint for eval
 runner (#28727)

---
 tools/caretaker-agent/Dockerfile              | 28 ++++++++++
 .../evals/triage/cloud_runner.py              | 39 ++++++++++++++
 .../evals/triage/helpers/sync_to_gcs.py       | 52 +++++++++++++++++++
 3 files changed, 119 insertions(+)
 create mode 100644 tools/caretaker-agent/Dockerfile
 create mode 100644 tools/caretaker-agent/evals/triage/cloud_runner.py
 create mode 100644 tools/caretaker-agent/evals/triage/helpers/sync_to_gcs.py

diff --git a/tools/caretaker-agent/Dockerfile b/tools/caretaker-agent/Dockerfile
new file mode 100644
index 0000000000..ced0e5da50
--- /dev/null
+++ b/tools/caretaker-agent/Dockerfile
@@ -0,0 +1,28 @@
+# ==============================================================================
+# Caretaker Triage Evaluation Runner Container (Cloud Run Job)
+#
+# Placed at repository root to allow `gcloud run jobs deploy --source .` to:
+# 1. Automatically detect this Dockerfile without separate build steps.
+# 2. Access both /cloudrun/triage-worker and /evals inside the root build context.
+# ==============================================================================
+
+FROM python:3.13-slim
+WORKDIR /app
+ENV PYTHONUNBUFFERED=1
+
+RUN apt-get update && apt-get install -y git curl && rm -rf /var/lib/apt/lists/*
+
+# 1. Pre-bake target gemini-cli repo clone into container image
+RUN git clone https://github.com/google-gemini/gemini-cli.git /app/evals/triage/target_repo
+
+# 2. Copy living local application code from root build context
+COPY cloudrun/triage-worker /app/cloudrun/triage-worker
+COPY evals /app/evals
+
+RUN pip install --no-cache-dir -r /app/cloudrun/triage-worker/requirements.txt \
+    && pip install --no-cache-dir -r /app/evals/triage/requirements.txt
+
+WORKDIR /app/evals/triage
+ENV PYTHONPATH=/app
+
+CMD ["python3", "cloud_runner.py"]
diff --git a/tools/caretaker-agent/evals/triage/cloud_runner.py b/tools/caretaker-agent/evals/triage/cloud_runner.py
new file mode 100644
index 0000000000..57e311adb7
--- /dev/null
+++ b/tools/caretaker-agent/evals/triage/cloud_runner.py
@@ -0,0 +1,39 @@
+"""
+Cloud Run Job Entrypoint for Gemini CLI Triage Evaluation Suite.
+Reads EVAL_CONFIG JSON environment variable, invokes run_suite(), and syncs results to GCS.
+"""
+
+import os
+import json
+from evals.triage.runner import run_suite
+from evals.triage.helpers.sync_to_gcs import sync_results_to_gcs
+
+
+def main() -> None:
+    config_str = os.environ.get("EVAL_CONFIG", "{}")
+    try:
+        cfg = json.loads(config_str) if config_str else {}
+        if not isinstance(cfg, dict):
+            raise ValueError(f"EVAL_CONFIG must be a JSON object, got {type(cfg).__name__}")
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Invalid EVAL_CONFIG JSON: {e}") from e
+
+    print("========================================================")
+    print(" 🚀 Running Gemini CLI Triage Evaluation Suite (Cloud Run)")
+    print("========================================================")
+    if cfg:
+        print(f"[EVAL_CONFIG] Loaded configuration: {cfg}")
+
+    # 1. Execute benchmark suite directly via run_suite()
+    run_suite(
+        filter_issues=cfg.get("issues"),
+        concurrency=cfg.get("concurrency", 5),
+        note=cfg.get("note")
+    )
+
+    # 2. Sync evaluation run results to GCS bucket
+    sync_results_to_gcs()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/caretaker-agent/evals/triage/helpers/sync_to_gcs.py b/tools/caretaker-agent/evals/triage/helpers/sync_to_gcs.py
new file mode 100644
index 0000000000..dec1d5ad5a
--- /dev/null
+++ b/tools/caretaker-agent/evals/triage/helpers/sync_to_gcs.py
@@ -0,0 +1,52 @@
+"""
+Helper script to sync evaluation run results from local container disk to GCS bucket.
+"""
+
+import os
+from google.cloud import storage
+
+
+def sync_results_to_gcs() -> None:
+    bucket_name = os.environ.get("EVAL_RESULTS_BUCKET", "triage-eval-results")
+    runs_dir = "results/runs"
+
+    if not os.path.exists(runs_dir):
+        print(f"⚠️ Warning: No '{runs_dir}' directory found to sync to GCS.")
+        return
+
+    print("\n========================================================")
+    print(" 📤 Syncing evaluation run results to GCS")
+    print(f" Bucket: gs://{bucket_name}/runs/")
+    print("========================================================")
+
+    try:
+        client = storage.Client()
+        bucket = client.bucket(bucket_name)
+        count = 0
+
+        run_folders = [d for d in os.listdir(runs_dir) if os.path.isdir(os.path.join(runs_dir, d))]
+        run_dest = f"gs://{bucket_name}/runs/{run_folders[0]}/" if run_folders else f"gs://{bucket_name}/runs/"
+
+        for root, _, files in os.walk(runs_dir):
+            for file in files:
+                local_path = os.path.join(root, file)
+                rel_path = os.path.relpath(local_path, runs_dir)
+                blob_path = f"runs/{rel_path}"
+                blob = bucket.blob(blob_path)
+                # Set explicit charset=utf-8 on GCS blobs so web browsers and Caretaker Dashboard render markdown emojis cleanly.
+                if file.endswith(".md"):
+                    blob.upload_from_filename(local_path, content_type="text/markdown; charset=utf-8")
+                elif file.endswith(".json"):
+                    blob.upload_from_filename(local_path, content_type="application/json; charset=utf-8")
+                else:
+                    blob.upload_from_filename(local_path)
+                count += 1
+
+        print(f"✅ Successfully uploaded {count} result artifact(s) to {run_dest}\n")
+    except Exception as e:
+        print(f"❌ Error: Failed to upload evaluation results to GCS: {e}")
+        raise
+
+
+if __name__ == "__main__":
+    sync_results_to_gcs()