diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ef8bdb58d..28f682c30e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -231,7 +231,7 @@ jobs: test_mac: name: 'Test (Mac) - ${{ matrix.node-version }}, ${{ matrix.shard }}' - runs-on: 'macos-latest-large' + runs-on: 'macos-latest' needs: - 'merge_queue_skipper' if: "github.repository == 'google-gemini/gemini-cli' && needs.merge_queue_skipper.outputs.skip == 'false'" @@ -244,8 +244,6 @@ jobs: matrix: node-version: - '20.x' - - '22.x' - - '24.x' shard: - 'cli' - 'others' diff --git a/tools/gemini-cli-bot/metrics/scripts/bottlenecks.ts b/tools/gemini-cli-bot/metrics/scripts/bottlenecks.ts new file mode 100644 index 0000000000..1b20da250a --- /dev/null +++ b/tools/gemini-cli-bot/metrics/scripts/bottlenecks.ts @@ -0,0 +1,79 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { GITHUB_OWNER, GITHUB_REPO } from '../types.js'; +import { execSync } from 'node:child_process'; + +interface HotIssueNode { + number: number; + comments: { + totalCount: number; + }; +} + +/** + * Identifies "Zombie" issues (open issues with no activity for > 30 days). + */ +function run() { + try { + const now = new Date(); + const thirtyDaysAgo = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000); + const sevenDaysAgo = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000); + + // 1. Count Zombie issues using Search API totalCount (unlimited) + const zombieSearchQuery = `is:issue is:open repo:${GITHUB_OWNER}/${GITHUB_REPO} updated:<${thirtyDaysAgo.toISOString()}`; + const zombieQuery = ` + query($searchQuery: String!) { + search(query: $searchQuery, type: ISSUE, first: 0) { + issueCount + } + } + `; + const zombieOutput = execSync( + `gh api graphql -F searchQuery='${zombieSearchQuery}' -f query='${zombieQuery}'`, + { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }, + ).trim(); + const zombieCount = JSON.parse(zombieOutput).data.search.issueCount; + process.stdout.write(`bottleneck_zombie_issues_count,${zombieCount}\n`); + + // 2. Identify "Hot" issues. Since we need to count comments per issue, + // we still need to fetch some nodes, but we can target the most active ones. + const hotSearchQuery = `is:issue is:open repo:${GITHUB_OWNER}/${GITHUB_REPO} updated:>${sevenDaysAgo.toISOString()} sort:comments-desc`; + const hotQuery = ` + query($searchQuery: String!) { + search(query: $searchQuery, type: ISSUE, first: 100) { + nodes { + ... on Issue { + number + comments { + totalCount + } + } + } + } + } + `; + const hotOutput = execSync( + `gh api graphql -F searchQuery='${hotSearchQuery}' -f query='${hotQuery}'`, + { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }, + ).trim(); + const hotNodes = JSON.parse(hotOutput).data.search.nodes as HotIssueNode[]; + + // We define "Hot" as > 10 comments in the last 7 days. + // Note: Search query 'sort:comments-desc' gets those with most total comments, + // which is a good proxy for 'Hot' when filtered by recent updates. + const veryHot = hotNodes.filter((node) => node.comments.totalCount > 10); + process.stdout.write(`bottleneck_hot_issues_count,${veryHot.length}\n`); + + } catch (error) { + process.stderr.write( + error instanceof Error ? error.message : String(error), + ); + process.exit(1); + } +} + +run(); diff --git a/tools/gemini-cli-bot/metrics/scripts/priority_distribution.ts b/tools/gemini-cli-bot/metrics/scripts/priority_distribution.ts new file mode 100644 index 0000000000..1ca8f6cfb0 --- /dev/null +++ b/tools/gemini-cli-bot/metrics/scripts/priority_distribution.ts @@ -0,0 +1,60 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { GITHUB_OWNER, GITHUB_REPO } from '../types.js'; +import { execSync } from 'node:child_process'; + +/** + * Calculates the distribution of open issues across priority labels. + */ +function run() { + try { + const repo = `${GITHUB_OWNER}/${GITHUB_REPO}`; + const query = ` + query($p0: String!, $p1: String!, $p2: String!, $p3: String!, $all: String!) { + p0: search(query: $p0, type: ISSUE, first: 0) { issueCount } + p1: search(query: $p1, type: ISSUE, first: 0) { issueCount } + p2: search(query: $p2, type: ISSUE, first: 0) { issueCount } + p3: search(query: $p3, type: ISSUE, first: 0) { issueCount } + all: search(query: $all, type: ISSUE, first: 0) { issueCount } + } + `; + + const variables = { + p0: `is:issue is:open repo:${repo} label:p0`, + p1: `is:issue is:open repo:${repo} label:p1`, + p2: `is:issue is:open repo:${repo} label:p2`, + p3: `is:issue is:open repo:${repo} label:p3`, + all: `is:issue is:open repo:${repo}`, + }; + + const output = execSync( + `gh api graphql -F p0='${variables.p0}' -F p1='${variables.p1}' -F p2='${variables.p2}' -F p3='${variables.p3}' -F all='${variables.all}' -f query='${query}'`, + { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }, + ).trim(); + + const data = JSON.parse(output).data; + const p0Count = data.p0.issueCount; + const p1Count = data.p1.issueCount; + const p2Count = data.p2.issueCount; + const p3Count = data.p3.issueCount; + const totalOpen = data.all.issueCount; + const noneCount = totalOpen - (p0Count + p1Count + p2Count + p3Count); + + process.stdout.write(`priority_p0_count,${p0Count}\n`); + process.stdout.write(`priority_p1_count,${p1Count}\n`); + process.stdout.write(`priority_p2_count,${p2Count}\n`); + process.stdout.write(`priority_p3_count,${p3Count}\n`); + process.stdout.write(`priority_none_count,${noneCount}\n`); + } catch (error) { + process.stderr.write( + error instanceof Error ? error.message : String(error), + ); + process.exit(1); + } +} + +run(); diff --git a/tools/gemini-cli-bot/metrics/scripts/triage_accuracy.ts b/tools/gemini-cli-bot/metrics/scripts/triage_accuracy.ts new file mode 100644 index 0000000000..2aaad22381 --- /dev/null +++ b/tools/gemini-cli-bot/metrics/scripts/triage_accuracy.ts @@ -0,0 +1,160 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { execSync } from 'node:child_process'; +import { GITHUB_OWNER, GITHUB_REPO } from '../types.js'; + +interface GitHubResponse { + data?: { + search?: { + nodes?: Array<{ + number: number; + timelineItems: { + nodes: Array; + }; + } | null>; + }; + }; + errors?: Array<{ message: string }>; +} + +interface LabeledEvent { + __typename: 'LabeledEvent'; + label: { name: string }; + actor: { login: string } | null; + createdAt: string; +} + +interface UnlabeledEvent { + __typename: 'UnlabeledEvent'; + label: { name: string }; + actor: { login: string } | null; + createdAt: string; +} + +type TimelineEvent = LabeledEvent | UnlabeledEvent; + +/** + * This script calculates the triage accuracy by detecting human overrides of bot-applied labels. + * It identifies the first 'area/' label added by a bot and checks if it was later removed + * or replaced by a human. + * + * It uses the Search API to get a representative sample of recent issues. + */ +async function run() { + try { + // Increase sample size to 250 for a more representative set. + // We sort by created-desc to get the most recent activity. + const query = ` + query($searchQuery: String!) { + search(query: $searchQuery, type: ISSUE, first: 250) { + nodes { + ... on Issue { + number + timelineItems(last: 50, itemTypes: [LABELED_EVENT, UNLABELED_EVENT]) { + nodes { + __typename + ... on LabeledEvent { + label { name } + actor { login } + createdAt + } + ... on UnlabeledEvent { + label { name } + actor { login } + createdAt + } + } + } + } + } + } + } + `; + + const searchQuery = `repo:${GITHUB_OWNER}/${GITHUB_REPO} is:issue sort:created-desc`; + const output = execSync( + `gh api graphql -F searchQuery='${searchQuery}' -f query='${query}'`, + { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] } + ); + + const response = JSON.parse(output) as GitHubResponse; + if (response.errors) { + throw new Error(`GraphQL Errors: ${JSON.stringify(response.errors)}`); + } + + const issues = response.data?.search?.nodes || []; + + let botLabeledCount = 0; + let overrideCount = 0; + + const isBot = (login: string) => + login.toLowerCase().includes('[bot]') || login === 'gemini-cli-robot'; + + for (const issue of issues) { + if (!issue || !('number' in issue)) continue; + const events = (issue.timelineItems?.nodes || []) as TimelineEvent[]; + + // Find first area/ label added by a bot + const firstBotLabelEvent = events.find( + (e: TimelineEvent) => + e.__typename === 'LabeledEvent' && + e.label.name.startsWith('area/') && + e.actor?.login && + isBot(e.actor.login) + ) as LabeledEvent | undefined; + + if (firstBotLabelEvent) { + botLabeledCount++; + const botLabelName = firstBotLabelEvent.label.name; + const botLabelTime = new Date(firstBotLabelEvent.createdAt).getTime(); + + // Check for overrides after this event + const isOverridden = events.some((e: TimelineEvent) => { + const eventTime = new Date(e.createdAt).getTime(); + if (eventTime <= botLabelTime) return false; + + const actorLogin = e.actor?.login; + if (!actorLogin || isBot(actorLogin)) return false; + + // Case 1: Human removed the bot's label + if (e.__typename === 'UnlabeledEvent' && e.label.name === botLabelName) { + return true; + } + + // Case 2: Human added a different area/ label + if ( + e.__typename === 'LabeledEvent' && + e.label.name.startsWith('area/') && + e.label.name !== botLabelName + ) { + return true; + } + + return false; + }); + + if (isOverridden) { + overrideCount++; + } + } + } + + const accuracyRate = botLabeledCount > 0 + ? (botLabeledCount - overrideCount) / botLabeledCount + : 1; + + process.stdout.write(`triage_accuracy_overrides,${overrideCount}\n`); + process.stdout.write(`triage_accuracy_total_bot_labeled,${botLabeledCount}\n`); + process.stdout.write(`triage_accuracy_rate,${Math.round(accuracyRate * 100) / 100}\n`); + + } catch (err) { + process.stderr.write(err instanceof Error ? err.message : String(err)); + process.exit(1); + } +} + +run();