diff --git a/tools/gemini-cli-bot/metrics/scripts/triage_accuracy.ts b/tools/gemini-cli-bot/metrics/scripts/triage_accuracy.ts index 560476f561..2aaad22381 100644 --- a/tools/gemini-cli-bot/metrics/scripts/triage_accuracy.ts +++ b/tools/gemini-cli-bot/metrics/scripts/triage_accuracy.ts @@ -7,18 +7,52 @@ import { execSync } from 'node:child_process'; import { GITHUB_OWNER, GITHUB_REPO } from '../types.js'; +interface GitHubResponse { + data?: { + search?: { + nodes?: Array<{ + number: number; + timelineItems: { + nodes: Array; + }; + } | null>; + }; + }; + errors?: Array<{ message: string }>; +} + +interface LabeledEvent { + __typename: 'LabeledEvent'; + label: { name: string }; + actor: { login: string } | null; + createdAt: string; +} + +interface UnlabeledEvent { + __typename: 'UnlabeledEvent'; + label: { name: string }; + actor: { login: string } | null; + createdAt: string; +} + +type TimelineEvent = LabeledEvent | UnlabeledEvent; + /** * This script calculates the triage accuracy by detecting human overrides of bot-applied labels. * It identifies the first 'area/' label added by a bot and checks if it was later removed * or replaced by a human. + * + * It uses the Search API to get a representative sample of recent issues. */ async function run() { try { + // Increase sample size to 250 for a more representative set. + // We sort by created-desc to get the most recent activity. const query = ` - query($owner: String!, $repo: String!) { - repository(owner: $owner, name: $repo) { - issues(last: 100, orderBy: {field: CREATED_AT, direction: ASC}) { - nodes { + query($searchQuery: String!) { + search(query: $searchQuery, type: ISSUE, first: 250) { + nodes { + ... on Issue { number timelineItems(last: 50, itemTypes: [LABELED_EVENT, UNLABELED_EVENT]) { nodes { @@ -41,17 +75,18 @@ async function run() { } `; + const searchQuery = `repo:${GITHUB_OWNER}/${GITHUB_REPO} is:issue sort:created-desc`; const output = execSync( - `gh api graphql -F owner=${GITHUB_OWNER} -F repo=${GITHUB_REPO} -f query='${query}'`, + `gh api graphql -F searchQuery='${searchQuery}' -f query='${query}'`, { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] } ); - const response = JSON.parse(output); + const response = JSON.parse(output) as GitHubResponse; if (response.errors) { throw new Error(`GraphQL Errors: ${JSON.stringify(response.errors)}`); } - const issues = response.data?.repository?.issues?.nodes || []; + const issues = response.data?.search?.nodes || []; let botLabeledCount = 0; let overrideCount = 0; @@ -60,17 +95,17 @@ async function run() { login.toLowerCase().includes('[bot]') || login === 'gemini-cli-robot'; for (const issue of issues) { - if (!issue) continue; - const events = (issue.timelineItems?.nodes || []) as any[]; + if (!issue || !('number' in issue)) continue; + const events = (issue.timelineItems?.nodes || []) as TimelineEvent[]; // Find first area/ label added by a bot const firstBotLabelEvent = events.find( - (e: any) => + (e: TimelineEvent) => e.__typename === 'LabeledEvent' && e.label.name.startsWith('area/') && e.actor?.login && isBot(e.actor.login) - ); + ) as LabeledEvent | undefined; if (firstBotLabelEvent) { botLabeledCount++; @@ -78,7 +113,7 @@ async function run() { const botLabelTime = new Date(firstBotLabelEvent.createdAt).getTime(); // Check for overrides after this event - const isOverridden = events.some((e: any) => { + const isOverridden = events.some((e: TimelineEvent) => { const eventTime = new Date(e.createdAt).getTime(); if (eventTime <= botLabelTime) return false;