mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-05-01 23:44:15 -07:00
# Metrics Integrity & Standardized Reporting (BT-01) (#26240)
Co-authored-by: gemini-cli[bot] <gemini-cli[bot]@users.noreply.github.com> Co-authored-by: Christian Gunderman <gundermanc@google.com>
This commit is contained in:
@@ -6,7 +6,7 @@
|
||||
* @license
|
||||
*/
|
||||
|
||||
import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js';
|
||||
import { GITHUB_OWNER, GITHUB_REPO } from '../types.js';
|
||||
import { execSync } from 'node:child_process';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
@@ -59,6 +59,27 @@ try {
|
||||
let totalMaintainerReviews = 0;
|
||||
let maintainerReviewsWithExpertise = 0;
|
||||
|
||||
// Cache git log authors per path to avoid redundant child_process calls
|
||||
const authorCache = new Map<string, string>();
|
||||
const getAuthors = (targetPath: string) => {
|
||||
if (authorCache.has(targetPath)) return authorCache.get(targetPath)!;
|
||||
try {
|
||||
const authors = execSync(
|
||||
`git log --format="%an|%ae" -- ${JSON.stringify(targetPath)}`,
|
||||
{
|
||||
cwd: repoRoot,
|
||||
encoding: 'utf-8',
|
||||
stdio: ['ignore', 'pipe', 'ignore'],
|
||||
},
|
||||
).toLowerCase();
|
||||
authorCache.set(targetPath, authors);
|
||||
return authors;
|
||||
} catch {
|
||||
authorCache.set(targetPath, '');
|
||||
return '';
|
||||
}
|
||||
};
|
||||
|
||||
for (const pr of data.pullRequests.nodes) {
|
||||
if (!pr.reviews?.nodes || pr.reviews.nodes.length === 0) continue;
|
||||
|
||||
@@ -73,31 +94,12 @@ try {
|
||||
const files = diffTreeOutput.split('\n').filter(Boolean);
|
||||
if (files.length === 0) continue;
|
||||
|
||||
// Cache git log authors per path to avoid redundant child_process calls
|
||||
const authorCache = new Map<string, string>();
|
||||
const getAuthors = (targetPath: string) => {
|
||||
if (authorCache.has(targetPath)) return authorCache.get(targetPath)!;
|
||||
try {
|
||||
const authors = execSync(
|
||||
`git log --format="%an|%ae" -- ${JSON.stringify(targetPath)}`,
|
||||
{
|
||||
cwd: repoRoot,
|
||||
encoding: 'utf-8',
|
||||
stdio: ['ignore', 'pipe', 'ignore'],
|
||||
},
|
||||
).toLowerCase();
|
||||
authorCache.set(targetPath, authors);
|
||||
return authors;
|
||||
} catch {
|
||||
authorCache.set(targetPath, '');
|
||||
return '';
|
||||
}
|
||||
};
|
||||
|
||||
const reviewersOnPR = new Map<string, { name?: string }>();
|
||||
for (const review of pr.reviews.nodes) {
|
||||
if (
|
||||
['MEMBER', 'OWNER'].includes(review.authorAssociation) &&
|
||||
['MEMBER', 'OWNER', 'COLLABORATOR'].includes(
|
||||
review.authorAssociation,
|
||||
) &&
|
||||
review.author?.login
|
||||
) {
|
||||
const login = review.author.login.toLowerCase();
|
||||
@@ -138,19 +140,8 @@ try {
|
||||
totalMaintainerReviews > 0
|
||||
? maintainerReviewsWithExpertise / totalMaintainerReviews
|
||||
: 0;
|
||||
const timestamp = new Date().toISOString();
|
||||
|
||||
process.stdout.write(
|
||||
JSON.stringify(<MetricOutput>{
|
||||
metric: 'domain_expertise',
|
||||
value: Math.round(ratio * 100) / 100,
|
||||
timestamp,
|
||||
details: {
|
||||
totalMaintainerReviews,
|
||||
maintainerReviewsWithExpertise,
|
||||
},
|
||||
}) + '\n',
|
||||
);
|
||||
process.stdout.write(`domain_expertise,${Math.round(ratio * 100) / 100}\n`);
|
||||
} catch (err) {
|
||||
process.stderr.write(err instanceof Error ? err.message : String(err));
|
||||
process.exit(1);
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* @license
|
||||
*/
|
||||
|
||||
import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js';
|
||||
import { GITHUB_OWNER, GITHUB_REPO } from '../types.js';
|
||||
import { execSync } from 'node:child_process';
|
||||
|
||||
try {
|
||||
@@ -96,42 +96,24 @@ try {
|
||||
);
|
||||
const issueOverall = calculateAvg(issues);
|
||||
|
||||
const timestamp = new Date().toISOString();
|
||||
|
||||
const metrics: MetricOutput[] = [
|
||||
{
|
||||
metric: 'latency_pr_overall_hours',
|
||||
value: Math.round(prOverall * 100) / 100,
|
||||
timestamp,
|
||||
},
|
||||
{
|
||||
metric: 'latency_pr_maintainers_hours',
|
||||
value: Math.round(prMaintainers * 100) / 100,
|
||||
timestamp,
|
||||
},
|
||||
{
|
||||
metric: 'latency_pr_community_hours',
|
||||
value: Math.round(prCommunity * 100) / 100,
|
||||
timestamp,
|
||||
},
|
||||
{
|
||||
metric: 'latency_issue_overall_hours',
|
||||
value: Math.round(issueOverall * 100) / 100,
|
||||
timestamp,
|
||||
},
|
||||
{
|
||||
metric: 'latency_issue_maintainers_hours',
|
||||
value: Math.round(issueMaintainers * 100) / 100,
|
||||
timestamp,
|
||||
},
|
||||
{
|
||||
metric: 'latency_issue_community_hours',
|
||||
value: Math.round(issueCommunity * 100) / 100,
|
||||
timestamp,
|
||||
},
|
||||
];
|
||||
|
||||
metrics.forEach((m) => process.stdout.write(JSON.stringify(m) + '\n'));
|
||||
process.stdout.write(
|
||||
`latency_pr_overall_hours,${Math.round(prOverall * 100) / 100}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`latency_pr_maintainers_hours,${Math.round(prMaintainers * 100) / 100}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`latency_pr_community_hours,${Math.round(prCommunity * 100) / 100}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`latency_issue_overall_hours,${Math.round(issueOverall * 100) / 100}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`latency_issue_maintainers_hours,${Math.round(issueMaintainers * 100) / 100}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`latency_issue_community_hours,${Math.round(issueCommunity * 100) / 100}\n`,
|
||||
);
|
||||
} catch (err) {
|
||||
process.stderr.write(err instanceof Error ? err.message : String(err));
|
||||
process.exit(1);
|
||||
|
||||
@@ -2,19 +2,31 @@
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* @license
|
||||
*/
|
||||
|
||||
import { execSync } from 'node:child_process';
|
||||
import { GITHUB_OWNER, GITHUB_REPO } from '../types.js';
|
||||
|
||||
try {
|
||||
const count = execSync(
|
||||
'gh issue list --state open --limit 1000 --json number --jq length',
|
||||
{
|
||||
encoding: 'utf-8',
|
||||
},
|
||||
const query = `
|
||||
query($owner: String!, $repo: String!) {
|
||||
repository(owner: $owner, name: $repo) {
|
||||
issues(states: OPEN) {
|
||||
totalCount
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
const output = execSync(
|
||||
`gh api graphql -F owner=${GITHUB_OWNER} -F repo=${GITHUB_REPO} -f query='${query}'`,
|
||||
{ encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] },
|
||||
).trim();
|
||||
console.log(`open_issues,${count}`);
|
||||
} catch {
|
||||
// Fallback if gh fails or no issues found
|
||||
console.log('open_issues,0');
|
||||
const parsed = JSON.parse(output);
|
||||
const totalCount = parsed?.data?.repository?.issues?.totalCount ?? 0;
|
||||
process.stdout.write(`open_issues,${totalCount}\n`);
|
||||
} catch (err) {
|
||||
process.stderr.write(err instanceof Error ? err.message : String(err));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -2,19 +2,31 @@
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* @license
|
||||
*/
|
||||
|
||||
import { execSync } from 'node:child_process';
|
||||
import { GITHUB_OWNER, GITHUB_REPO } from '../types.js';
|
||||
|
||||
try {
|
||||
const count = execSync(
|
||||
'gh pr list --state open --limit 1000 --json number --jq length',
|
||||
{
|
||||
encoding: 'utf-8',
|
||||
},
|
||||
const query = `
|
||||
query($owner: String!, $repo: String!) {
|
||||
repository(owner: $owner, name: $repo) {
|
||||
pullRequests(states: OPEN) {
|
||||
totalCount
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
const output = execSync(
|
||||
`gh api graphql -F owner=${GITHUB_OWNER} -F repo=${GITHUB_REPO} -f query='${query}'`,
|
||||
{ encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] },
|
||||
).trim();
|
||||
console.log(`open_prs,${count}`);
|
||||
} catch {
|
||||
// Fallback if gh fails or no PRs found
|
||||
console.log('open_prs,0');
|
||||
const parsed = JSON.parse(output);
|
||||
const totalCount = parsed?.data?.repository?.pullRequests?.totalCount ?? 0;
|
||||
process.stdout.write(`open_prs,${totalCount}\n`);
|
||||
} catch (err) {
|
||||
process.stderr.write(err instanceof Error ? err.message : String(err));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* @license
|
||||
*/
|
||||
|
||||
import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js';
|
||||
import { GITHUB_OWNER, GITHUB_REPO } from '../types.js';
|
||||
import { execSync } from 'node:child_process';
|
||||
|
||||
try {
|
||||
@@ -41,7 +41,9 @@ try {
|
||||
|
||||
for (const review of pr.reviews.nodes) {
|
||||
if (
|
||||
['MEMBER', 'OWNER'].includes(review.authorAssociation) &&
|
||||
['MEMBER', 'OWNER', 'COLLABORATOR'].includes(
|
||||
review.authorAssociation,
|
||||
) &&
|
||||
review.author?.login
|
||||
) {
|
||||
const login = review.author.login.toLowerCase();
|
||||
@@ -66,15 +68,8 @@ try {
|
||||
counts.reduce((a, b) => a + Math.pow(b - mean, 2), 0) / counts.length;
|
||||
}
|
||||
|
||||
const timestamp = new Date().toISOString();
|
||||
|
||||
process.stdout.write(
|
||||
JSON.stringify(<MetricOutput>{
|
||||
metric: 'review_distribution_variance',
|
||||
value: Math.round(variance * 100) / 100,
|
||||
timestamp,
|
||||
details: reviewCounts,
|
||||
}) + '\n',
|
||||
`review_distribution_variance,${Math.round(variance * 100) / 100}\n`,
|
||||
);
|
||||
} catch (err) {
|
||||
process.stderr.write(err instanceof Error ? err.message : String(err));
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* @license
|
||||
*/
|
||||
|
||||
import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js';
|
||||
import { GITHUB_OWNER, GITHUB_REPO } from '../types.js';
|
||||
import { execSync } from 'node:child_process';
|
||||
|
||||
try {
|
||||
@@ -87,61 +87,33 @@ try {
|
||||
),
|
||||
);
|
||||
|
||||
const timestamp = new Date().toISOString();
|
||||
|
||||
const metrics: MetricOutput[] = [
|
||||
{
|
||||
metric: 'throughput_pr_overall_per_day',
|
||||
value: Math.round(prOverall * 100) / 100,
|
||||
timestamp,
|
||||
},
|
||||
{
|
||||
metric: 'throughput_pr_maintainers_per_day',
|
||||
value: Math.round(prMaintainers * 100) / 100,
|
||||
timestamp,
|
||||
},
|
||||
{
|
||||
metric: 'throughput_pr_community_per_day',
|
||||
value: Math.round(prCommunity * 100) / 100,
|
||||
timestamp,
|
||||
},
|
||||
{
|
||||
metric: 'throughput_issue_overall_per_day',
|
||||
value: Math.round(issueOverall * 100) / 100,
|
||||
timestamp,
|
||||
},
|
||||
{
|
||||
metric: 'throughput_issue_maintainers_per_day',
|
||||
value: Math.round(issueMaintainers * 100) / 100,
|
||||
timestamp,
|
||||
},
|
||||
{
|
||||
metric: 'throughput_issue_community_per_day',
|
||||
value: Math.round(issueCommunity * 100) / 100,
|
||||
timestamp,
|
||||
},
|
||||
{
|
||||
metric: 'throughput_issue_overall_days_per_issue',
|
||||
value: issueOverall > 0 ? Math.round((1 / issueOverall) * 100) / 100 : 0,
|
||||
timestamp,
|
||||
},
|
||||
{
|
||||
metric: 'throughput_issue_maintainers_days_per_issue',
|
||||
value:
|
||||
issueMaintainers > 0
|
||||
? Math.round((1 / issueMaintainers) * 100) / 100
|
||||
: 0,
|
||||
timestamp,
|
||||
},
|
||||
{
|
||||
metric: 'throughput_issue_community_days_per_issue',
|
||||
value:
|
||||
issueCommunity > 0 ? Math.round((1 / issueCommunity) * 100) / 100 : 0,
|
||||
timestamp,
|
||||
},
|
||||
];
|
||||
|
||||
metrics.forEach((m) => process.stdout.write(JSON.stringify(m) + '\n'));
|
||||
process.stdout.write(
|
||||
`throughput_pr_overall_per_day,${Math.round(prOverall * 100) / 100}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`throughput_pr_maintainers_per_day,${Math.round(prMaintainers * 100) / 100}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`throughput_pr_community_per_day,${Math.round(prCommunity * 100) / 100}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`throughput_issue_overall_per_day,${Math.round(issueOverall * 100) / 100}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`throughput_issue_maintainers_per_day,${Math.round(issueMaintainers * 100) / 100}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`throughput_issue_community_per_day,${Math.round(issueCommunity * 100) / 100}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`throughput_issue_overall_days_per_issue,${issueOverall > 0 ? Math.round((1 / issueOverall) * 100) / 100 : 0}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`throughput_issue_maintainers_days_per_issue,${issueMaintainers > 0 ? Math.round((1 / issueMaintainers) * 100) / 100 : 0}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`throughput_issue_community_days_per_issue,${issueCommunity > 0 ? Math.round((1 / issueCommunity) * 100) / 100 : 0}\n`,
|
||||
);
|
||||
} catch (err) {
|
||||
process.stderr.write(err instanceof Error ? err.message : String(err));
|
||||
process.exit(1);
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* @license
|
||||
*/
|
||||
|
||||
import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js';
|
||||
import { GITHUB_OWNER, GITHUB_REPO } from '../types.js';
|
||||
import { execSync } from 'node:child_process';
|
||||
|
||||
try {
|
||||
@@ -118,8 +118,8 @@ try {
|
||||
const issues = processItems(data.issues.nodes);
|
||||
const allItems = [...prs, ...issues];
|
||||
|
||||
const isMaintainer = (assoc: string) => ['MEMBER', 'OWNER'].includes(assoc);
|
||||
const is1P = (assoc: string) => ['COLLABORATOR'].includes(assoc);
|
||||
const isMaintainer = (assoc: string) =>
|
||||
['MEMBER', 'OWNER', 'COLLABORATOR'].includes(assoc);
|
||||
|
||||
const calculateAvg = (items: { ttfr: number; association: string }[]) =>
|
||||
items.length ? items.reduce((a, b) => a + b.ttfr, 0) / items.length : 0;
|
||||
@@ -127,30 +127,14 @@ try {
|
||||
const maintainers = calculateAvg(
|
||||
allItems.filter((i) => isMaintainer(i.association)),
|
||||
);
|
||||
const firstParty = calculateAvg(allItems.filter((i) => is1P(i.association)));
|
||||
const overall = calculateAvg(allItems);
|
||||
|
||||
const timestamp = new Date().toISOString();
|
||||
|
||||
const metrics: MetricOutput[] = [
|
||||
{
|
||||
metric: 'time_to_first_response_overall_hours',
|
||||
value: Math.round(overall * 100) / 100,
|
||||
timestamp,
|
||||
},
|
||||
{
|
||||
metric: 'time_to_first_response_maintainers_hours',
|
||||
value: Math.round(maintainers * 100) / 100,
|
||||
timestamp,
|
||||
},
|
||||
{
|
||||
metric: 'time_to_first_response_1p_hours',
|
||||
value: Math.round(firstParty * 100) / 100,
|
||||
timestamp,
|
||||
},
|
||||
];
|
||||
|
||||
metrics.forEach((m) => process.stdout.write(JSON.stringify(m) + '\n'));
|
||||
process.stdout.write(
|
||||
`time_to_first_response_overall_hours,${Math.round(overall * 100) / 100}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
`time_to_first_response_maintainers_hours,${Math.round(maintainers * 100) / 100}\n`,
|
||||
);
|
||||
} catch (err) {
|
||||
process.stderr.write(err instanceof Error ? err.message : String(err));
|
||||
process.exit(1);
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* @license
|
||||
*/
|
||||
|
||||
import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js';
|
||||
import { GITHUB_OWNER, GITHUB_REPO } from '../types.js';
|
||||
import { execSync } from 'node:child_process';
|
||||
|
||||
try {
|
||||
@@ -71,28 +71,14 @@ try {
|
||||
allItems.filter((i) => !isMaintainer(i.association)),
|
||||
);
|
||||
|
||||
const timestamp = new Date().toISOString();
|
||||
|
||||
process.stdout.write(
|
||||
JSON.stringify(<MetricOutput>{
|
||||
metric: 'user_touches_overall',
|
||||
value: Math.round(overall * 100) / 100,
|
||||
timestamp,
|
||||
}) + '\n',
|
||||
`user_touches_overall,${Math.round(overall * 100) / 100}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
JSON.stringify(<MetricOutput>{
|
||||
metric: 'user_touches_maintainers',
|
||||
value: Math.round(maintainers * 100) / 100,
|
||||
timestamp,
|
||||
}) + '\n',
|
||||
`user_touches_maintainers,${Math.round(maintainers * 100) / 100}\n`,
|
||||
);
|
||||
process.stdout.write(
|
||||
JSON.stringify(<MetricOutput>{
|
||||
metric: 'user_touches_community',
|
||||
value: Math.round(community * 100) / 100,
|
||||
timestamp,
|
||||
}) + '\n',
|
||||
`user_touches_community,${Math.round(community * 100) / 100}\n`,
|
||||
);
|
||||
} catch (err) {
|
||||
process.stderr.write(err instanceof Error ? err.message : String(err));
|
||||
|
||||
Reference in New Issue
Block a user