diff --git a/.github/workflows/gemini-automated-issue-triage.yml b/.github/workflows/gemini-automated-issue-triage.yml index 18a8b076ea..00cc9d338e 100644 --- a/.github/workflows/gemini-automated-issue-triage.yml +++ b/.github/workflows/gemini-automated-issue-triage.yml @@ -263,6 +263,7 @@ jobs: - Do NOT include any introductory or concluding remarks, explanations, or additional text. - Do NOT include any thoughts or reasoning outside the JSON block. - Ensure the output is a single JSON object with a "labels_to_set" array. + - Return the JSON as your final text response. Do NOT use `run_shell_command` or `echo` to print it. - name: 'Apply Labels to Issue' if: |- diff --git a/.github/workflows/gemini-scheduled-issue-triage.yml b/.github/workflows/gemini-scheduled-issue-triage.yml index 9b19974afc..d62b892692 100644 --- a/.github/workflows/gemini-scheduled-issue-triage.yml +++ b/.github/workflows/gemini-scheduled-issue-triage.yml @@ -221,6 +221,7 @@ jobs: - Do NOT include any introductory or concluding remarks, explanations, or additional text. - Do NOT include any thoughts or reasoning outside the JSON block. - Ensure the output is a single JSON array of objects. + - Return the JSON as your final text response. Do NOT use `run_shell_command` or `echo` to print it. - name: 'Apply Labels to Issues' if: |- diff --git a/evals/batch_triage.eval.ts b/evals/batch_triage.eval.ts index 609dab9f57..52376e7bc2 100644 --- a/evals/batch_triage.eval.ts +++ b/evals/batch_triage.eval.ts @@ -103,14 +103,24 @@ const assertHasIssueLabel = (issueNumber: number, expectedLabel: string) => { // The model response JSON is in the 'response' field const responseText = output.response; - const firstBrace = responseText.indexOf('['); - const lastBrace = responseText.lastIndexOf(']'); - if (firstBrace === -1 || lastBrace === -1 || lastBrace < firstBrace) { - throw new Error( - `Could not find a JSON array in the response: "${escapeHtml(responseText)}"`, - ); + let jsonString: string; + const match = responseText.match(/```json\s*([\s\S]*?)\s*```/); + if (match?.[1]) { + jsonString = match[1]; + } else { + const firstBracket = responseText.indexOf('['); + const lastBracket = responseText.lastIndexOf(']'); + if ( + firstBracket === -1 || + lastBracket === -1 || + lastBracket < firstBracket + ) { + throw new Error( + `Could not find a JSON array in the response: "${escapeHtml(responseText)}"`, + ); + } + jsonString = responseText.substring(firstBracket, lastBracket + 1); } - const jsonString = responseText.substring(firstBrace, lastBrace + 1); let data: { issue_number: number; labels_to_add: string[] }[]; try { diff --git a/evals/triage.eval.ts b/evals/triage.eval.ts index 4b6b2b80f3..d32087d4dc 100644 --- a/evals/triage.eval.ts +++ b/evals/triage.eval.ts @@ -103,14 +103,20 @@ const assertHasLabel = (expectedLabel: string) => { // The model response JSON is in the 'response' field const responseText = output.response; - const firstBrace = responseText.indexOf('{'); - const lastBrace = responseText.lastIndexOf('}'); - if (firstBrace === -1 || lastBrace === -1 || lastBrace < firstBrace) { - throw new Error( - `Could not find a JSON object in the response: "${escapeHtml(responseText)}"`, - ); + let jsonString: string; + const match = responseText.match(/```json\s*([\s\S]*?)\s*```/); + if (match?.[1]) { + jsonString = match[1]; + } else { + const firstBrace = responseText.indexOf('{'); + const lastBrace = responseText.lastIndexOf('}'); + if (firstBrace === -1 || lastBrace === -1 || lastBrace < firstBrace) { + throw new Error( + `Could not find a JSON object in the response: "${escapeHtml(responseText)}"`, + ); + } + jsonString = responseText.substring(firstBrace, lastBrace + 1); } - const jsonString = responseText.substring(firstBrace, lastBrace + 1); let data: { labels_to_set?: string[] }; try {