From caa04664161331e8e3751fe4c5e5fd30198e9e09 Mon Sep 17 00:00:00 2001 From: "gemini-cli[bot]" <218312386+gemini-cli[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 16:11:39 -0700 Subject: [PATCH] # Metrics Integrity & Standardized Reporting (BT-01) (#26240) Co-authored-by: gemini-cli[bot] Co-authored-by: Christian Gunderman --- .../metrics/scripts/domain_expertise.ts | 61 ++++++-------- .../gemini-cli-bot/metrics/scripts/latency.ts | 56 +++++-------- .../metrics/scripts/open_issues.ts | 30 +++++-- .../metrics/scripts/open_prs.ts | 30 +++++-- .../metrics/scripts/review_distribution.ts | 15 ++-- .../metrics/scripts/throughput.ts | 84 +++++++------------ .../metrics/scripts/time_to_first_response.ts | 34 ++------ .../metrics/scripts/user_touches.ts | 22 +---- 8 files changed, 133 insertions(+), 199 deletions(-) diff --git a/tools/gemini-cli-bot/metrics/scripts/domain_expertise.ts b/tools/gemini-cli-bot/metrics/scripts/domain_expertise.ts index e4b72099ee..a35d8d5648 100644 --- a/tools/gemini-cli-bot/metrics/scripts/domain_expertise.ts +++ b/tools/gemini-cli-bot/metrics/scripts/domain_expertise.ts @@ -6,7 +6,7 @@ * @license */ -import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js'; +import { GITHUB_OWNER, GITHUB_REPO } from '../types.js'; import { execSync } from 'node:child_process'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -59,6 +59,27 @@ try { let totalMaintainerReviews = 0; let maintainerReviewsWithExpertise = 0; + // Cache git log authors per path to avoid redundant child_process calls + const authorCache = new Map(); + const getAuthors = (targetPath: string) => { + if (authorCache.has(targetPath)) return authorCache.get(targetPath)!; + try { + const authors = execSync( + `git log --format="%an|%ae" -- ${JSON.stringify(targetPath)}`, + { + cwd: repoRoot, + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'ignore'], + }, + ).toLowerCase(); + authorCache.set(targetPath, authors); + return authors; + } catch { + authorCache.set(targetPath, ''); + return ''; + } + }; + for (const pr of data.pullRequests.nodes) { if (!pr.reviews?.nodes || pr.reviews.nodes.length === 0) continue; @@ -73,31 +94,12 @@ try { const files = diffTreeOutput.split('\n').filter(Boolean); if (files.length === 0) continue; - // Cache git log authors per path to avoid redundant child_process calls - const authorCache = new Map(); - const getAuthors = (targetPath: string) => { - if (authorCache.has(targetPath)) return authorCache.get(targetPath)!; - try { - const authors = execSync( - `git log --format="%an|%ae" -- ${JSON.stringify(targetPath)}`, - { - cwd: repoRoot, - encoding: 'utf-8', - stdio: ['ignore', 'pipe', 'ignore'], - }, - ).toLowerCase(); - authorCache.set(targetPath, authors); - return authors; - } catch { - authorCache.set(targetPath, ''); - return ''; - } - }; - const reviewersOnPR = new Map(); for (const review of pr.reviews.nodes) { if ( - ['MEMBER', 'OWNER'].includes(review.authorAssociation) && + ['MEMBER', 'OWNER', 'COLLABORATOR'].includes( + review.authorAssociation, + ) && review.author?.login ) { const login = review.author.login.toLowerCase(); @@ -138,19 +140,8 @@ try { totalMaintainerReviews > 0 ? maintainerReviewsWithExpertise / totalMaintainerReviews : 0; - const timestamp = new Date().toISOString(); - process.stdout.write( - JSON.stringify({ - metric: 'domain_expertise', - value: Math.round(ratio * 100) / 100, - timestamp, - details: { - totalMaintainerReviews, - maintainerReviewsWithExpertise, - }, - }) + '\n', - ); + process.stdout.write(`domain_expertise,${Math.round(ratio * 100) / 100}\n`); } catch (err) { process.stderr.write(err instanceof Error ? err.message : String(err)); process.exit(1); diff --git a/tools/gemini-cli-bot/metrics/scripts/latency.ts b/tools/gemini-cli-bot/metrics/scripts/latency.ts index b96201a51d..7dd5dcd1f6 100644 --- a/tools/gemini-cli-bot/metrics/scripts/latency.ts +++ b/tools/gemini-cli-bot/metrics/scripts/latency.ts @@ -6,7 +6,7 @@ * @license */ -import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js'; +import { GITHUB_OWNER, GITHUB_REPO } from '../types.js'; import { execSync } from 'node:child_process'; try { @@ -96,42 +96,24 @@ try { ); const issueOverall = calculateAvg(issues); - const timestamp = new Date().toISOString(); - - const metrics: MetricOutput[] = [ - { - metric: 'latency_pr_overall_hours', - value: Math.round(prOverall * 100) / 100, - timestamp, - }, - { - metric: 'latency_pr_maintainers_hours', - value: Math.round(prMaintainers * 100) / 100, - timestamp, - }, - { - metric: 'latency_pr_community_hours', - value: Math.round(prCommunity * 100) / 100, - timestamp, - }, - { - metric: 'latency_issue_overall_hours', - value: Math.round(issueOverall * 100) / 100, - timestamp, - }, - { - metric: 'latency_issue_maintainers_hours', - value: Math.round(issueMaintainers * 100) / 100, - timestamp, - }, - { - metric: 'latency_issue_community_hours', - value: Math.round(issueCommunity * 100) / 100, - timestamp, - }, - ]; - - metrics.forEach((m) => process.stdout.write(JSON.stringify(m) + '\n')); + process.stdout.write( + `latency_pr_overall_hours,${Math.round(prOverall * 100) / 100}\n`, + ); + process.stdout.write( + `latency_pr_maintainers_hours,${Math.round(prMaintainers * 100) / 100}\n`, + ); + process.stdout.write( + `latency_pr_community_hours,${Math.round(prCommunity * 100) / 100}\n`, + ); + process.stdout.write( + `latency_issue_overall_hours,${Math.round(issueOverall * 100) / 100}\n`, + ); + process.stdout.write( + `latency_issue_maintainers_hours,${Math.round(issueMaintainers * 100) / 100}\n`, + ); + process.stdout.write( + `latency_issue_community_hours,${Math.round(issueCommunity * 100) / 100}\n`, + ); } catch (err) { process.stderr.write(err instanceof Error ? err.message : String(err)); process.exit(1); diff --git a/tools/gemini-cli-bot/metrics/scripts/open_issues.ts b/tools/gemini-cli-bot/metrics/scripts/open_issues.ts index 4996ec7ce4..93a9877898 100644 --- a/tools/gemini-cli-bot/metrics/scripts/open_issues.ts +++ b/tools/gemini-cli-bot/metrics/scripts/open_issues.ts @@ -2,19 +2,31 @@ * @license * Copyright 2026 Google LLC * SPDX-License-Identifier: Apache-2.0 + * + * @license */ import { execSync } from 'node:child_process'; +import { GITHUB_OWNER, GITHUB_REPO } from '../types.js'; try { - const count = execSync( - 'gh issue list --state open --limit 1000 --json number --jq length', - { - encoding: 'utf-8', - }, + const query = ` + query($owner: String!, $repo: String!) { + repository(owner: $owner, name: $repo) { + issues(states: OPEN) { + totalCount + } + } + } + `; + const output = execSync( + `gh api graphql -F owner=${GITHUB_OWNER} -F repo=${GITHUB_REPO} -f query='${query}'`, + { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }, ).trim(); - console.log(`open_issues,${count}`); -} catch { - // Fallback if gh fails or no issues found - console.log('open_issues,0'); + const parsed = JSON.parse(output); + const totalCount = parsed?.data?.repository?.issues?.totalCount ?? 0; + process.stdout.write(`open_issues,${totalCount}\n`); +} catch (err) { + process.stderr.write(err instanceof Error ? err.message : String(err)); + process.exit(1); } diff --git a/tools/gemini-cli-bot/metrics/scripts/open_prs.ts b/tools/gemini-cli-bot/metrics/scripts/open_prs.ts index 35819ef0f9..fdab545b9d 100644 --- a/tools/gemini-cli-bot/metrics/scripts/open_prs.ts +++ b/tools/gemini-cli-bot/metrics/scripts/open_prs.ts @@ -2,19 +2,31 @@ * @license * Copyright 2026 Google LLC * SPDX-License-Identifier: Apache-2.0 + * + * @license */ import { execSync } from 'node:child_process'; +import { GITHUB_OWNER, GITHUB_REPO } from '../types.js'; try { - const count = execSync( - 'gh pr list --state open --limit 1000 --json number --jq length', - { - encoding: 'utf-8', - }, + const query = ` + query($owner: String!, $repo: String!) { + repository(owner: $owner, name: $repo) { + pullRequests(states: OPEN) { + totalCount + } + } + } + `; + const output = execSync( + `gh api graphql -F owner=${GITHUB_OWNER} -F repo=${GITHUB_REPO} -f query='${query}'`, + { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }, ).trim(); - console.log(`open_prs,${count}`); -} catch { - // Fallback if gh fails or no PRs found - console.log('open_prs,0'); + const parsed = JSON.parse(output); + const totalCount = parsed?.data?.repository?.pullRequests?.totalCount ?? 0; + process.stdout.write(`open_prs,${totalCount}\n`); +} catch (err) { + process.stderr.write(err instanceof Error ? err.message : String(err)); + process.exit(1); } diff --git a/tools/gemini-cli-bot/metrics/scripts/review_distribution.ts b/tools/gemini-cli-bot/metrics/scripts/review_distribution.ts index 05f6b71740..eda4044916 100644 --- a/tools/gemini-cli-bot/metrics/scripts/review_distribution.ts +++ b/tools/gemini-cli-bot/metrics/scripts/review_distribution.ts @@ -6,7 +6,7 @@ * @license */ -import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js'; +import { GITHUB_OWNER, GITHUB_REPO } from '../types.js'; import { execSync } from 'node:child_process'; try { @@ -41,7 +41,9 @@ try { for (const review of pr.reviews.nodes) { if ( - ['MEMBER', 'OWNER'].includes(review.authorAssociation) && + ['MEMBER', 'OWNER', 'COLLABORATOR'].includes( + review.authorAssociation, + ) && review.author?.login ) { const login = review.author.login.toLowerCase(); @@ -66,15 +68,8 @@ try { counts.reduce((a, b) => a + Math.pow(b - mean, 2), 0) / counts.length; } - const timestamp = new Date().toISOString(); - process.stdout.write( - JSON.stringify({ - metric: 'review_distribution_variance', - value: Math.round(variance * 100) / 100, - timestamp, - details: reviewCounts, - }) + '\n', + `review_distribution_variance,${Math.round(variance * 100) / 100}\n`, ); } catch (err) { process.stderr.write(err instanceof Error ? err.message : String(err)); diff --git a/tools/gemini-cli-bot/metrics/scripts/throughput.ts b/tools/gemini-cli-bot/metrics/scripts/throughput.ts index 3a259aaefb..3806dd407a 100644 --- a/tools/gemini-cli-bot/metrics/scripts/throughput.ts +++ b/tools/gemini-cli-bot/metrics/scripts/throughput.ts @@ -6,7 +6,7 @@ * @license */ -import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js'; +import { GITHUB_OWNER, GITHUB_REPO } from '../types.js'; import { execSync } from 'node:child_process'; try { @@ -87,61 +87,33 @@ try { ), ); - const timestamp = new Date().toISOString(); - - const metrics: MetricOutput[] = [ - { - metric: 'throughput_pr_overall_per_day', - value: Math.round(prOverall * 100) / 100, - timestamp, - }, - { - metric: 'throughput_pr_maintainers_per_day', - value: Math.round(prMaintainers * 100) / 100, - timestamp, - }, - { - metric: 'throughput_pr_community_per_day', - value: Math.round(prCommunity * 100) / 100, - timestamp, - }, - { - metric: 'throughput_issue_overall_per_day', - value: Math.round(issueOverall * 100) / 100, - timestamp, - }, - { - metric: 'throughput_issue_maintainers_per_day', - value: Math.round(issueMaintainers * 100) / 100, - timestamp, - }, - { - metric: 'throughput_issue_community_per_day', - value: Math.round(issueCommunity * 100) / 100, - timestamp, - }, - { - metric: 'throughput_issue_overall_days_per_issue', - value: issueOverall > 0 ? Math.round((1 / issueOverall) * 100) / 100 : 0, - timestamp, - }, - { - metric: 'throughput_issue_maintainers_days_per_issue', - value: - issueMaintainers > 0 - ? Math.round((1 / issueMaintainers) * 100) / 100 - : 0, - timestamp, - }, - { - metric: 'throughput_issue_community_days_per_issue', - value: - issueCommunity > 0 ? Math.round((1 / issueCommunity) * 100) / 100 : 0, - timestamp, - }, - ]; - - metrics.forEach((m) => process.stdout.write(JSON.stringify(m) + '\n')); + process.stdout.write( + `throughput_pr_overall_per_day,${Math.round(prOverall * 100) / 100}\n`, + ); + process.stdout.write( + `throughput_pr_maintainers_per_day,${Math.round(prMaintainers * 100) / 100}\n`, + ); + process.stdout.write( + `throughput_pr_community_per_day,${Math.round(prCommunity * 100) / 100}\n`, + ); + process.stdout.write( + `throughput_issue_overall_per_day,${Math.round(issueOverall * 100) / 100}\n`, + ); + process.stdout.write( + `throughput_issue_maintainers_per_day,${Math.round(issueMaintainers * 100) / 100}\n`, + ); + process.stdout.write( + `throughput_issue_community_per_day,${Math.round(issueCommunity * 100) / 100}\n`, + ); + process.stdout.write( + `throughput_issue_overall_days_per_issue,${issueOverall > 0 ? Math.round((1 / issueOverall) * 100) / 100 : 0}\n`, + ); + process.stdout.write( + `throughput_issue_maintainers_days_per_issue,${issueMaintainers > 0 ? Math.round((1 / issueMaintainers) * 100) / 100 : 0}\n`, + ); + process.stdout.write( + `throughput_issue_community_days_per_issue,${issueCommunity > 0 ? Math.round((1 / issueCommunity) * 100) / 100 : 0}\n`, + ); } catch (err) { process.stderr.write(err instanceof Error ? err.message : String(err)); process.exit(1); diff --git a/tools/gemini-cli-bot/metrics/scripts/time_to_first_response.ts b/tools/gemini-cli-bot/metrics/scripts/time_to_first_response.ts index fde2a6346b..3585d6443b 100644 --- a/tools/gemini-cli-bot/metrics/scripts/time_to_first_response.ts +++ b/tools/gemini-cli-bot/metrics/scripts/time_to_first_response.ts @@ -6,7 +6,7 @@ * @license */ -import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js'; +import { GITHUB_OWNER, GITHUB_REPO } from '../types.js'; import { execSync } from 'node:child_process'; try { @@ -118,8 +118,8 @@ try { const issues = processItems(data.issues.nodes); const allItems = [...prs, ...issues]; - const isMaintainer = (assoc: string) => ['MEMBER', 'OWNER'].includes(assoc); - const is1P = (assoc: string) => ['COLLABORATOR'].includes(assoc); + const isMaintainer = (assoc: string) => + ['MEMBER', 'OWNER', 'COLLABORATOR'].includes(assoc); const calculateAvg = (items: { ttfr: number; association: string }[]) => items.length ? items.reduce((a, b) => a + b.ttfr, 0) / items.length : 0; @@ -127,30 +127,14 @@ try { const maintainers = calculateAvg( allItems.filter((i) => isMaintainer(i.association)), ); - const firstParty = calculateAvg(allItems.filter((i) => is1P(i.association))); const overall = calculateAvg(allItems); - const timestamp = new Date().toISOString(); - - const metrics: MetricOutput[] = [ - { - metric: 'time_to_first_response_overall_hours', - value: Math.round(overall * 100) / 100, - timestamp, - }, - { - metric: 'time_to_first_response_maintainers_hours', - value: Math.round(maintainers * 100) / 100, - timestamp, - }, - { - metric: 'time_to_first_response_1p_hours', - value: Math.round(firstParty * 100) / 100, - timestamp, - }, - ]; - - metrics.forEach((m) => process.stdout.write(JSON.stringify(m) + '\n')); + process.stdout.write( + `time_to_first_response_overall_hours,${Math.round(overall * 100) / 100}\n`, + ); + process.stdout.write( + `time_to_first_response_maintainers_hours,${Math.round(maintainers * 100) / 100}\n`, + ); } catch (err) { process.stderr.write(err instanceof Error ? err.message : String(err)); process.exit(1); diff --git a/tools/gemini-cli-bot/metrics/scripts/user_touches.ts b/tools/gemini-cli-bot/metrics/scripts/user_touches.ts index 192897479b..5ccffa94fc 100644 --- a/tools/gemini-cli-bot/metrics/scripts/user_touches.ts +++ b/tools/gemini-cli-bot/metrics/scripts/user_touches.ts @@ -6,7 +6,7 @@ * @license */ -import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js'; +import { GITHUB_OWNER, GITHUB_REPO } from '../types.js'; import { execSync } from 'node:child_process'; try { @@ -71,28 +71,14 @@ try { allItems.filter((i) => !isMaintainer(i.association)), ); - const timestamp = new Date().toISOString(); - process.stdout.write( - JSON.stringify({ - metric: 'user_touches_overall', - value: Math.round(overall * 100) / 100, - timestamp, - }) + '\n', + `user_touches_overall,${Math.round(overall * 100) / 100}\n`, ); process.stdout.write( - JSON.stringify({ - metric: 'user_touches_maintainers', - value: Math.round(maintainers * 100) / 100, - timestamp, - }) + '\n', + `user_touches_maintainers,${Math.round(maintainers * 100) / 100}\n`, ); process.stdout.write( - JSON.stringify({ - metric: 'user_touches_community', - value: Math.round(community * 100) / 100, - timestamp, - }) + '\n', + `user_touches_community,${Math.round(community * 100) / 100}\n`, ); } catch (err) { process.stderr.write(err instanceof Error ? err.message : String(err));