fix(triage): tune prompt to return json text and harden eval json extraction

.github/workflows/gemini-automated-issue-triage.yml

@@ -263,6 +263,7 @@ jobs:
             - Do NOT include any introductory or concluding remarks, explanations, or additional text.
             - Do NOT include any thoughts or reasoning outside the JSON block.
             - Ensure the output is a single JSON object with a "labels_to_set" array.
+            - Return the JSON as your final text response. Do NOT use `run_shell_command` or `echo` to print it.
 
       - name: 'Apply Labels to Issue'
         if: |-

.github/workflows/gemini-scheduled-issue-triage.yml

@@ -221,6 +221,7 @@ jobs:
             - Do NOT include any introductory or concluding remarks, explanations, or additional text.
             - Do NOT include any thoughts or reasoning outside the JSON block.
             - Ensure the output is a single JSON array of objects.
+            - Return the JSON as your final text response. Do NOT use `run_shell_command` or `echo` to print it.
 
       - name: 'Apply Labels to Issues'
         if: |-

evals/batch_triage.eval.ts

@@ -103,14 +103,24 @@ const assertHasIssueLabel = (issueNumber: number, expectedLabel: string) => {
 
     // The model response JSON is in the 'response' field
     const responseText = output.response;
-    const firstBrace = responseText.indexOf('[');
+    let jsonString: string;
-    const lastBrace = responseText.lastIndexOf(']');
+    const match = responseText.match(/```json\s*([\s\S]*?)\s*```/);
-    if (firstBrace === -1 || lastBrace === -1 || lastBrace < firstBrace) {
+    if (match?.[1]) {
-      throw new Error(
+      jsonString = match[1];
-        `Could not find a JSON array in the response: "${escapeHtml(responseText)}"`,
+    } else {
-      );
+      const firstBracket = responseText.indexOf('[');
+      const lastBracket = responseText.lastIndexOf(']');
+      if (
+        firstBracket === -1 ||
+        lastBracket === -1 ||
+        lastBracket < firstBracket
+      ) {
+        throw new Error(
+          `Could not find a JSON array in the response: "${escapeHtml(responseText)}"`,
+        );
+      }
+      jsonString = responseText.substring(firstBracket, lastBracket + 1);
     }
-    const jsonString = responseText.substring(firstBrace, lastBrace + 1);
 
     let data: { issue_number: number; labels_to_add: string[] }[];
     try {

evals/triage.eval.ts

@@ -103,14 +103,20 @@ const assertHasLabel = (expectedLabel: string) => {
 
     // The model response JSON is in the 'response' field
     const responseText = output.response;
-    const firstBrace = responseText.indexOf('{');
+    let jsonString: string;
-    const lastBrace = responseText.lastIndexOf('}');
+    const match = responseText.match(/```json\s*([\s\S]*?)\s*```/);
-    if (firstBrace === -1 || lastBrace === -1 || lastBrace < firstBrace) {
+    if (match?.[1]) {
-      throw new Error(
+      jsonString = match[1];
-        `Could not find a JSON object in the response: "${escapeHtml(responseText)}"`,
+    } else {
-      );
+      const firstBrace = responseText.indexOf('{');
+      const lastBrace = responseText.lastIndexOf('}');
+      if (firstBrace === -1 || lastBrace === -1 || lastBrace < firstBrace) {
+        throw new Error(
+          `Could not find a JSON object in the response: "${escapeHtml(responseText)}"`,
+        );
+      }
+      jsonString = responseText.substring(firstBrace, lastBrace + 1);
     }
-    const jsonString = responseText.substring(firstBrace, lastBrace + 1);
 
     let data: { labels_to_set?: string[] };
     try {