gemini-cli/tools/gemini-cli-bot/metrics/scripts/triage_accuracy.ts

/**
 * @license
 * Copyright 2026 Google LLC
 * SPDX-License-Identifier: Apache-2.0
 */

import { execSync } from 'node:child_process';
import { GITHUB_OWNER, GITHUB_REPO } from '../types.js';

/**
 * This script calculates the triage accuracy by detecting human overrides of bot-applied labels.
 * It identifies the first 'area/' label added by a bot and checks if it was later removed
 * or replaced by a human.
 */
async function run() {
  try {
    const query = `
    query($owner: String!, $repo: String!) {
      repository(owner: $owner, name: $repo) {
        issues(last: 100, orderBy: {field: CREATED_AT, direction: ASC}) {
          nodes {
            number
            timelineItems(last: 50, itemTypes: [LABELED_EVENT, UNLABELED_EVENT]) {
              nodes {
                __typename
                ... on LabeledEvent {
                  label { name }
                  actor { login }
                  createdAt
                }
                ... on UnlabeledEvent {
                  label { name }
                  actor { login }
                  createdAt
                }
              }
            }
          }
        }
      }
    }
    `;

    const output = execSync(
      `gh api graphql -F owner=${GITHUB_OWNER} -F repo=${GITHUB_REPO} -f query='${query}'`,
      { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }
    );

    const response = JSON.parse(output);
    if (response.errors) {
      throw new Error(`GraphQL Errors: ${JSON.stringify(response.errors)}`);
    }

    const issues = response.data?.repository?.issues?.nodes || [];

    let botLabeledCount = 0;
    let overrideCount = 0;

    const isBot = (login: string) =>
      login.toLowerCase().includes('[bot]') || login === 'gemini-cli-robot';

    for (const issue of issues) {
      if (!issue) continue;
      const events = (issue.timelineItems?.nodes || []) as any[];
      
      // Find first area/ label added by a bot
      const firstBotLabelEvent = events.find(
        (e: any) =>
          e.__typename === 'LabeledEvent' &&
          e.label.name.startsWith('area/') &&
          e.actor?.login &&
          isBot(e.actor.login)
      );

      if (firstBotLabelEvent) {
        botLabeledCount++;
        const botLabelName = firstBotLabelEvent.label.name;
        const botLabelTime = new Date(firstBotLabelEvent.createdAt).getTime();

        // Check for overrides after this event
        const isOverridden = events.some((e: any) => {
          const eventTime = new Date(e.createdAt).getTime();
          if (eventTime <= botLabelTime) return false;

          const actorLogin = e.actor?.login;
          if (!actorLogin || isBot(actorLogin)) return false;

          // Case 1: Human removed the bot's label
          if (e.__typename === 'UnlabeledEvent' && e.label.name === botLabelName) {
            return true;
          }

          // Case 2: Human added a different area/ label
          if (
            e.__typename === 'LabeledEvent' &&
            e.label.name.startsWith('area/') &&
            e.label.name !== botLabelName
          ) {
            return true;
          }

          return false;
        });

        if (isOverridden) {
          overrideCount++;
        }
      }
    }

    const accuracyRate = botLabeledCount > 0
      ? (botLabeledCount - overrideCount) / botLabeledCount
      : 1;

    process.stdout.write(`triage_accuracy_overrides,${overrideCount}\n`);
    process.stdout.write(`triage_accuracy_total_bot_labeled,${botLabeledCount}\n`);
    process.stdout.write(`triage_accuracy_rate,${Math.round(accuracyRate * 100) / 100}\n`);

  } catch (err) {
    process.stderr.write(err instanceof Error ? err.message : String(err));
    process.exit(1);
  }
}

run();