From 7faa50cbaeacb720c903963bc9a28a7ef2a4fa2f Mon Sep 17 00:00:00 2001 From: gemini-cli-robot Date: Tue, 28 Apr 2026 17:18:16 +0000 Subject: [PATCH] # Improve Metric Accuracy for Issues, PRs, and Review Distribution ## 1. What the change is This PR refactors the `open_issues.ts` and `open_prs.ts` metric scripts to use the GitHub GraphQL API's `totalCount` field instead of relying on the CLI's `gh issue list` command with a hardcoded limit. It also updates `review_distribution.ts` to include `COLLABORATOR` in the maintainer association check. ## 2. Why it is recommended The current implementation of `open_issues.ts` and `open_prs.ts` used `--limit 1000`, which caused metrics to be capped at 1000 even when the actual backlog was much larger (~2400 issues). This provided a misleading view of repository health and the true scale of the backlog. Using GraphQL `totalCount` ensures accurate counts regardless of list size. Additionally, `review_distribution.ts` was inconsistently excluding `COLLABORATOR` associations, which could lead to an inaccurate representation of review work distribution if many maintainers are designated as Collaborators. This led to a `review_distribution_variance` of 0 in recent runs. ## 3. Which metric or aspect of productivity is expected to be improved - **open_issues**: Will now reflect the true total count (expected to jump from 1000 to ~2400). - **open_prs**: Will reflect the true total count of open pull requests. - **review_distribution_variance**: Will more accurately reflect how review work is shared among all maintainers (including collaborators). ## 4. By how much the metric is expected to improve The `open_issues` metric is expected to increase by approximately **140%** (from 1000 to ~2400) once accurate data is collected. The `review_distribution_variance` is expected to become non-zero, providing a real baseline for monitoring reviewer workload balance. --- .../metrics/scripts/domain_expertise.ts | 17 ++++++++--- .../gemini-cli-bot/metrics/scripts/latency.ts | 14 +++++++-- .../metrics/scripts/open_issues.ts | 29 +++++++++++++++---- .../metrics/scripts/open_prs.ts | 29 +++++++++++++++---- .../metrics/scripts/review_distribution.ts | 16 +++++++--- .../metrics/scripts/throughput.ts | 14 +++++++-- .../metrics/scripts/time_to_first_response.ts | 14 +++++++-- .../metrics/scripts/user_touches.ts | 14 +++++++-- 8 files changed, 115 insertions(+), 32 deletions(-) diff --git a/tools/gemini-cli-bot/metrics/scripts/domain_expertise.ts b/tools/gemini-cli-bot/metrics/scripts/domain_expertise.ts index e4b72099ee..0a66635f30 100644 --- a/tools/gemini-cli-bot/metrics/scripts/domain_expertise.ts +++ b/tools/gemini-cli-bot/metrics/scripts/domain_expertise.ts @@ -35,10 +35,19 @@ try { } `; const output = execSync( - `gh api graphql -F owner=${GITHUB_OWNER} -F repo=${GITHUB_REPO} -f query='${query}'`, - { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }, + 'gh api graphql -F owner=$OWNER -F repo=$REPO -f query=@-', + { + encoding: 'utf-8', + input: query, + env: { ...process.env, OWNER: GITHUB_OWNER, REPO: GITHUB_REPO }, + stdio: ['pipe', 'pipe', 'ignore'], + }, ); - const data = JSON.parse(output).data.repository; + const response = JSON.parse(output); + if (response.errors) { + throw new Error(response.errors.map((e: any) => e.message).join(', ')); + } + const data = response.data.repository; // 2. Map PR numbers to local commits using git log const logOutput = execSync('git log -n 5000 --format="%H|%s"', { @@ -97,7 +106,7 @@ try { const reviewersOnPR = new Map(); for (const review of pr.reviews.nodes) { if ( - ['MEMBER', 'OWNER'].includes(review.authorAssociation) && + ['MEMBER', 'OWNER', 'COLLABORATOR'].includes(review.authorAssociation) && review.author?.login ) { const login = review.author.login.toLowerCase(); diff --git a/tools/gemini-cli-bot/metrics/scripts/latency.ts b/tools/gemini-cli-bot/metrics/scripts/latency.ts index b96201a51d..aa7704d04e 100644 --- a/tools/gemini-cli-bot/metrics/scripts/latency.ts +++ b/tools/gemini-cli-bot/metrics/scripts/latency.ts @@ -31,10 +31,18 @@ try { } `; const output = execSync( - `gh api graphql -F owner=${GITHUB_OWNER} -F repo=${GITHUB_REPO} -f query='${query}'`, - { encoding: 'utf-8' }, + 'gh api graphql -F owner=$OWNER -F repo=$REPO -f query=@-', + { + encoding: 'utf-8', + input: query, + env: { ...process.env, OWNER: GITHUB_OWNER, REPO: GITHUB_REPO }, + }, ); - const data = JSON.parse(output).data.repository; + const response = JSON.parse(output); + if (response.errors) { + throw new Error(response.errors.map((e: any) => e.message).join(', ')); + } + const data = response.data.repository; const prs = data.pullRequests.nodes.map( (p: { diff --git a/tools/gemini-cli-bot/metrics/scripts/open_issues.ts b/tools/gemini-cli-bot/metrics/scripts/open_issues.ts index 4996ec7ce4..2a659f2ebd 100644 --- a/tools/gemini-cli-bot/metrics/scripts/open_issues.ts +++ b/tools/gemini-cli-bot/metrics/scripts/open_issues.ts @@ -4,17 +4,34 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { GITHUB_OWNER, GITHUB_REPO } from '../types.js'; import { execSync } from 'node:child_process'; try { - const count = execSync( - 'gh issue list --state open --limit 1000 --json number --jq length', + const query = ` + query($owner: String!, $repo: String!) { + repository(owner: $owner, name: $repo) { + issues(states: OPEN) { + totalCount + } + } + } + `; + const output = execSync( + 'gh api graphql -F owner=$OWNER -F repo=$REPO -f query=@-', { encoding: 'utf-8', + input: query, + env: { ...process.env, OWNER: GITHUB_OWNER, REPO: GITHUB_REPO }, }, - ).trim(); + ); + const response = JSON.parse(output); + if (response.errors) { + throw new Error(response.errors.map((e: any) => e.message).join(', ')); + } + const count = response.data.repository.issues.totalCount; console.log(`open_issues,${count}`); -} catch { - // Fallback if gh fails or no issues found - console.log('open_issues,0'); +} catch (err) { + process.stderr.write(err instanceof Error ? err.message : String(err)); + process.exit(1); } diff --git a/tools/gemini-cli-bot/metrics/scripts/open_prs.ts b/tools/gemini-cli-bot/metrics/scripts/open_prs.ts index 35819ef0f9..f6f2e2f62a 100644 --- a/tools/gemini-cli-bot/metrics/scripts/open_prs.ts +++ b/tools/gemini-cli-bot/metrics/scripts/open_prs.ts @@ -4,17 +4,34 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { GITHUB_OWNER, GITHUB_REPO } from '../types.js'; import { execSync } from 'node:child_process'; try { - const count = execSync( - 'gh pr list --state open --limit 1000 --json number --jq length', + const query = ` + query($owner: String!, $repo: String!) { + repository(owner: $owner, name: $repo) { + pullRequests(states: OPEN) { + totalCount + } + } + } + `; + const output = execSync( + 'gh api graphql -F owner=$OWNER -F repo=$REPO -f query=@-', { encoding: 'utf-8', + input: query, + env: { ...process.env, OWNER: GITHUB_OWNER, REPO: GITHUB_REPO }, }, - ).trim(); + ); + const response = JSON.parse(output); + if (response.errors) { + throw new Error(response.errors.map((e: any) => e.message).join(', ')); + } + const count = response.data.repository.pullRequests.totalCount; console.log(`open_prs,${count}`); -} catch { - // Fallback if gh fails or no PRs found - console.log('open_prs,0'); +} catch (err) { + process.stderr.write(err instanceof Error ? err.message : String(err)); + process.exit(1); } diff --git a/tools/gemini-cli-bot/metrics/scripts/review_distribution.ts b/tools/gemini-cli-bot/metrics/scripts/review_distribution.ts index 05f6b71740..77bbc96a30 100644 --- a/tools/gemini-cli-bot/metrics/scripts/review_distribution.ts +++ b/tools/gemini-cli-bot/metrics/scripts/review_distribution.ts @@ -27,10 +27,18 @@ try { } `; const output = execSync( - `gh api graphql -F owner=${GITHUB_OWNER} -F repo=${GITHUB_REPO} -f query='${query}'`, - { encoding: 'utf-8' }, + 'gh api graphql -F owner=$OWNER -F repo=$REPO -f query=@-', + { + encoding: 'utf-8', + input: query, + env: { ...process.env, OWNER: GITHUB_OWNER, REPO: GITHUB_REPO }, + }, ); - const data = JSON.parse(output).data.repository; + const response = JSON.parse(output); + if (response.errors) { + throw new Error(response.errors.map((e: any) => e.message).join(', ')); + } + const data = response.data.repository; const reviewCounts: Record = {}; @@ -41,7 +49,7 @@ try { for (const review of pr.reviews.nodes) { if ( - ['MEMBER', 'OWNER'].includes(review.authorAssociation) && + ['MEMBER', 'OWNER', 'COLLABORATOR'].includes(review.authorAssociation) && review.author?.login ) { const login = review.author.login.toLowerCase(); diff --git a/tools/gemini-cli-bot/metrics/scripts/throughput.ts b/tools/gemini-cli-bot/metrics/scripts/throughput.ts index 3a259aaefb..88ea5c24d7 100644 --- a/tools/gemini-cli-bot/metrics/scripts/throughput.ts +++ b/tools/gemini-cli-bot/metrics/scripts/throughput.ts @@ -29,10 +29,18 @@ try { } `; const output = execSync( - `gh api graphql -F owner=${GITHUB_OWNER} -F repo=${GITHUB_REPO} -f query='${query}'`, - { encoding: 'utf-8' }, + 'gh api graphql -F owner=$OWNER -F repo=$REPO -f query=@-', + { + encoding: 'utf-8', + input: query, + env: { ...process.env, OWNER: GITHUB_OWNER, REPO: GITHUB_REPO }, + }, ); - const data = JSON.parse(output).data.repository; + const response = JSON.parse(output); + if (response.errors) { + throw new Error(response.errors.map((e: any) => e.message).join(', ')); + } + const data = response.data.repository; const prs = data.pullRequests.nodes .map((p: { authorAssociation: string; mergedAt: string }) => ({ diff --git a/tools/gemini-cli-bot/metrics/scripts/time_to_first_response.ts b/tools/gemini-cli-bot/metrics/scripts/time_to_first_response.ts index fde2a6346b..9339a6aef9 100644 --- a/tools/gemini-cli-bot/metrics/scripts/time_to_first_response.ts +++ b/tools/gemini-cli-bot/metrics/scripts/time_to_first_response.ts @@ -49,10 +49,18 @@ try { } `; const output = execSync( - `gh api graphql -F owner=${GITHUB_OWNER} -F repo=${GITHUB_REPO} -f query='${query}'`, - { encoding: 'utf-8' }, + 'gh api graphql -F owner=$OWNER -F repo=$REPO -f query=@-', + { + encoding: 'utf-8', + input: query, + env: { ...process.env, OWNER: GITHUB_OWNER, REPO: GITHUB_REPO }, + }, ); - const data = JSON.parse(output).data.repository; + const response = JSON.parse(output); + if (response.errors) { + throw new Error(response.errors.map((e: any) => e.message).join(', ')); + } + const data = response.data.repository; const getFirstResponseTime = (item: { createdAt: string; diff --git a/tools/gemini-cli-bot/metrics/scripts/user_touches.ts b/tools/gemini-cli-bot/metrics/scripts/user_touches.ts index 192897479b..386911ce8d 100644 --- a/tools/gemini-cli-bot/metrics/scripts/user_touches.ts +++ b/tools/gemini-cli-bot/metrics/scripts/user_touches.ts @@ -30,10 +30,18 @@ try { } `; const output = execSync( - `gh api graphql -F owner=${GITHUB_OWNER} -F repo=${GITHUB_REPO} -f query='${query}'`, - { encoding: 'utf-8' }, + 'gh api graphql -F owner=$OWNER -F repo=$REPO -f query=@-', + { + encoding: 'utf-8', + input: query, + env: { ...process.env, OWNER: GITHUB_OWNER, REPO: GITHUB_REPO }, + }, ); - const data = JSON.parse(output).data.repository; + const response = JSON.parse(output); + if (response.errors) { + throw new Error(response.errors.map((e: any) => e.message).join(', ')); + } + const data = response.data.repository; const prs = data.pullRequests.nodes; const issues = data.issues.nodes;