From 6f4dff177a3ad7b730fb8653bca55fb117e51486 Mon Sep 17 00:00:00 2001 From: Christian Gunderman Date: Fri, 24 Apr 2026 11:31:24 -0700 Subject: [PATCH] Record as time-series. --- .github/workflows/gemini-cli-bot-pulse.yml | 6 ++ tools/gemini-cli-bot/history/sync.ts | 98 +++++++++++++++++++ .../gemini-cli-bot/metrics/history-helper.ts | 61 ++++++++++++ tools/gemini-cli-bot/metrics/index.ts | 85 +++++++++++++++- .../metrics/scripts/domain_expertise.ts | 2 +- .../gemini-cli-bot/metrics/scripts/latency.ts | 2 +- .../metrics/scripts/review_distribution.ts | 2 +- .../metrics/scripts/throughput.ts | 2 +- .../metrics/scripts/time_to_first_response.ts | 2 +- 9 files changed, 251 insertions(+), 9 deletions(-) create mode 100644 tools/gemini-cli-bot/history/sync.ts create mode 100644 tools/gemini-cli-bot/metrics/history-helper.ts diff --git a/.github/workflows/gemini-cli-bot-pulse.yml b/.github/workflows/gemini-cli-bot-pulse.yml index 0fdd04aeec..b3af9f2f46 100644 --- a/.github/workflows/gemini-cli-bot-pulse.yml +++ b/.github/workflows/gemini-cli-bot-pulse.yml @@ -45,6 +45,12 @@ jobs: name: 'metrics-before' path: 'metrics-before.csv' + - name: 'Archive Time-series' + uses: 'actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02' # ratchet:actions/upload-artifact@v4 + with: + name: 'metrics-timeseries' + path: 'tools/gemini-cli-bot/history/metrics-timeseries.csv' + - name: 'Run Reflex Processes' env: GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' diff --git a/tools/gemini-cli-bot/history/sync.ts b/tools/gemini-cli-bot/history/sync.ts new file mode 100644 index 0000000000..114c032e95 --- /dev/null +++ b/tools/gemini-cli-bot/history/sync.ts @@ -0,0 +1,98 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { execSync } from 'node:child_process'; +import { + writeFileSync, + readFileSync, + existsSync, + mkdirSync, + rmSync, +} from 'node:fs'; +import { join } from 'node:path'; + +const HISTORY_DIR = join(process.cwd(), 'tools', 'gemini-cli-bot', 'history'); +const WORKFLOW = 'gemini-cli-bot-pulse.yml'; + +function runCommand(command: string): string { + try { + return execSync(command, { + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); + } catch { + return ''; + } +} + +async function sync() { + if (!existsSync(HISTORY_DIR)) { + mkdirSync(HISTORY_DIR, { recursive: true }); + } + + console.log('Searching for previous successful Pulse run...'); + const runId = runCommand( + `gh run list --workflow ${WORKFLOW} --status success --limit 1 --json databaseId --jq '.[0].databaseId'`, + ); + + if (!runId) { + console.log('No previous successful run found.'); + return; + } + + console.log(`Found run ${runId}. Downloading artifacts...`); + + const tempDir = join(HISTORY_DIR, 'temp_dl'); + if (existsSync(tempDir)) { + rmSync(tempDir, { recursive: true, force: true }); + } + mkdirSync(tempDir, { recursive: true }); + + // Download metrics-timeseries if it exists + try { + execSync(`gh run download ${runId} -n metrics-timeseries -D ${tempDir}`, { + stdio: 'ignore', + }); + const tsFile = join(tempDir, 'metrics-timeseries.csv'); + if (existsSync(tsFile)) { + writeFileSync( + join(HISTORY_DIR, 'metrics-timeseries.csv'), + readFileSync(tsFile), + ); + console.log('Downloaded metrics-timeseries.csv'); + } + } catch { + console.log('metrics-timeseries artifact not found in previous run.'); + } + + // Download previous metrics-before.csv + try { + execSync(`gh run download ${runId} -n metrics-before -D ${tempDir}`, { + stdio: 'ignore', + }); + const mbFile = join(tempDir, 'metrics-before.csv'); + if (existsSync(mbFile)) { + writeFileSync( + join(HISTORY_DIR, 'metrics-before-prev.csv'), + readFileSync(mbFile), + ); + console.log( + 'Downloaded previous metrics-before.csv as metrics-before-prev.csv', + ); + } + } catch { + console.log('metrics-before artifact not found in previous run.'); + } + + // Clean up + rmSync(tempDir, { recursive: true, force: true }); +} + +sync().catch((error) => { + console.error('Error syncing history:', error); + // Don't fail the whole process if sync fails + process.exit(0); +}); diff --git a/tools/gemini-cli-bot/metrics/history-helper.ts b/tools/gemini-cli-bot/metrics/history-helper.ts new file mode 100644 index 0000000000..5c4c607f18 --- /dev/null +++ b/tools/gemini-cli-bot/metrics/history-helper.ts @@ -0,0 +1,61 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { readFileSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; + +const TIMESERIES_FILE = join( + process.cwd(), + 'tools', + 'gemini-cli-bot', + 'history', + 'metrics-timeseries.csv', +); + +/** + * Calculates the historical average of a metric over a given number of days. + */ +export function getHistoricalAverage( + metric: string, + days: number, +): number | null { + if (!existsSync(TIMESERIES_FILE)) return null; + + try { + const content = readFileSync(TIMESERIES_FILE, 'utf-8'); + const lines = content.split('\n').slice(1); // skip header + const now = new Date(); + const threshold = new Date(now.getTime() - days * 24 * 60 * 60 * 1000); + + const values: number[] = []; + for (const line of lines) { + if (!line.trim()) continue; + const parts = line.split(','); + if (parts.length < 3) continue; + + const timestamp = parts[0]; + const m = parts[1]; + const value = parts[2]; + + if (m === metric) { + const date = new Date(timestamp); + if (date >= threshold) { + const numValue = parseFloat(value); + if (!isNaN(numValue)) { + values.push(numValue); + } + } + } + } + + if (values.length === 0) return null; + const sum = values.reduce((a, b) => a + b, 0); + return sum / values.length; + } catch (error) { + console.error(`Error reading historical average for ${metric}:`, error); + return null; + } +} diff --git a/tools/gemini-cli-bot/metrics/index.ts b/tools/gemini-cli-bot/metrics/index.ts index e65ffba0c3..b76df43b53 100644 --- a/tools/gemini-cli-bot/metrics/index.ts +++ b/tools/gemini-cli-bot/metrics/index.ts @@ -4,9 +4,10 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { readdirSync, writeFileSync } from 'node:fs'; +import { readdirSync, writeFileSync, existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; import { execSync } from 'node:child_process'; +import { getHistoricalAverage } from './history-helper.js'; const SCRIPTS_DIR = join( process.cwd(), @@ -15,12 +16,29 @@ const SCRIPTS_DIR = join( 'metrics', 'scripts', ); +const SYNC_SCRIPT = join( + process.cwd(), + 'tools', + 'gemini-cli-bot', + 'history', + 'sync.ts', +); const OUTPUT_FILE = join(process.cwd(), 'metrics-before.csv'); +const TIMESERIES_FILE = join( + process.cwd(), + 'tools', + 'gemini-cli-bot', + 'history', + 'metrics-timeseries.csv', +); function processOutputLine(line: string, results: string[]) { const trimmedLine = line.trim(); if (!trimmedLine) return; + let metricName = ''; + let metricValue = 0; + try { const parsed = JSON.parse(trimmedLine); if ( @@ -29,16 +47,59 @@ function processOutputLine(line: string, results: string[]) { 'metric' in parsed && 'value' in parsed ) { - results.push(`${parsed.metric},${parsed.value}`); + metricName = parsed.metric; + metricValue = parseFloat(parsed.value); + results.push(`${metricName},${metricValue}`); } else { - results.push(trimmedLine); + const parts = trimmedLine.split(','); + if (parts.length === 2) { + metricName = parts[0]; + metricValue = parseFloat(parts[1]); + results.push(trimmedLine); + } else { + results.push(trimmedLine); + return; // Unable to parse for deltas + } } } catch { - results.push(trimmedLine); + const parts = trimmedLine.split(','); + if (parts.length === 2) { + metricName = parts[0]; + metricValue = parseFloat(parts[1]); + results.push(trimmedLine); + } else { + results.push(trimmedLine); + return; // Unable to parse for deltas + } + } + + // Calculate and append deltas if the metric is a valid number + if (metricName && !isNaN(metricValue)) { + const avg7d = getHistoricalAverage(metricName, 7); + if (avg7d !== null) { + results.push( + `${metricName}_delta_7d,${(metricValue - avg7d).toFixed(2)}`, + ); + } + + const avg30d = getHistoricalAverage(metricName, 30); + if (avg30d !== null) { + results.push( + `${metricName}_delta_30d,${(metricValue - avg30d).toFixed(2)}`, + ); + } } } async function run() { + // Sync history first + console.log('Syncing history...'); + try { + execSync(`npx tsx ${JSON.stringify(SYNC_SCRIPT)}`, { stdio: 'inherit' }); + } catch (error) { + console.error('History sync failed, continuing without history:', error); + } + const scripts = readdirSync(SCRIPTS_DIR).filter( (file) => file.endsWith('.ts') || file.endsWith('.js'), ); @@ -64,6 +125,22 @@ async function run() { writeFileSync(OUTPUT_FILE, results.join('\n')); console.log(`Saved metrics to ${OUTPUT_FILE}`); + + // Update timeseries + const timestamp = new Date().toISOString(); + let timeseriesContent = ''; + if (existsSync(TIMESERIES_FILE)) { + timeseriesContent = readFileSync(TIMESERIES_FILE, 'utf-8').trim(); + } else { + timeseriesContent = 'timestamp,metric,value'; + } + + const newRows = results.slice(1).map((row) => `${timestamp},${row}`); + if (newRows.length > 0) { + timeseriesContent += '\n' + newRows.join('\n'); + writeFileSync(TIMESERIES_FILE, timeseriesContent + '\n'); + console.log(`Updated timeseries at ${TIMESERIES_FILE}`); + } } run().catch(console.error); diff --git a/tools/gemini-cli-bot/metrics/scripts/domain_expertise.ts b/tools/gemini-cli-bot/metrics/scripts/domain_expertise.ts index 637892617e..e4b72099ee 100644 --- a/tools/gemini-cli-bot/metrics/scripts/domain_expertise.ts +++ b/tools/gemini-cli-bot/metrics/scripts/domain_expertise.ts @@ -6,7 +6,7 @@ * @license */ -import { GITHUB_OWNER, GITHUB_REPO, MetricOutput } from '../types.js'; +import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js'; import { execSync } from 'node:child_process'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; diff --git a/tools/gemini-cli-bot/metrics/scripts/latency.ts b/tools/gemini-cli-bot/metrics/scripts/latency.ts index c8b461c8bd..b96201a51d 100644 --- a/tools/gemini-cli-bot/metrics/scripts/latency.ts +++ b/tools/gemini-cli-bot/metrics/scripts/latency.ts @@ -6,7 +6,7 @@ * @license */ -import { GITHUB_OWNER, GITHUB_REPO, MetricOutput } from '../types.js'; +import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js'; import { execSync } from 'node:child_process'; try { diff --git a/tools/gemini-cli-bot/metrics/scripts/review_distribution.ts b/tools/gemini-cli-bot/metrics/scripts/review_distribution.ts index e62fa99945..05f6b71740 100644 --- a/tools/gemini-cli-bot/metrics/scripts/review_distribution.ts +++ b/tools/gemini-cli-bot/metrics/scripts/review_distribution.ts @@ -6,7 +6,7 @@ * @license */ -import { GITHUB_OWNER, GITHUB_REPO, MetricOutput } from '../types.js'; +import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js'; import { execSync } from 'node:child_process'; try { diff --git a/tools/gemini-cli-bot/metrics/scripts/throughput.ts b/tools/gemini-cli-bot/metrics/scripts/throughput.ts index 5f5a6f57f3..3a259aaefb 100644 --- a/tools/gemini-cli-bot/metrics/scripts/throughput.ts +++ b/tools/gemini-cli-bot/metrics/scripts/throughput.ts @@ -6,7 +6,7 @@ * @license */ -import { GITHUB_OWNER, GITHUB_REPO, MetricOutput } from '../types.js'; +import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js'; import { execSync } from 'node:child_process'; try { diff --git a/tools/gemini-cli-bot/metrics/scripts/time_to_first_response.ts b/tools/gemini-cli-bot/metrics/scripts/time_to_first_response.ts index 7241802932..fde2a6346b 100644 --- a/tools/gemini-cli-bot/metrics/scripts/time_to_first_response.ts +++ b/tools/gemini-cli-bot/metrics/scripts/time_to_first_response.ts @@ -6,7 +6,7 @@ * @license */ -import { GITHUB_OWNER, GITHUB_REPO, MetricOutput } from '../types.js'; +import { GITHUB_OWNER, GITHUB_REPO, type MetricOutput } from '../types.js'; import { execSync } from 'node:child_process'; try {