Files
gemini-cli/.github/workflows/ci.yml
T
gemini-cli[bot] b4f49bf37a perf: Implement Search-Based Velocity Metrics and Optimize Mac CI
This PR resolves the critical "2,160 items/day" throughput reporting anomaly and reduces CI costs by optimizing macOS runners.

### Changes:
- **Search-Based Sampling**: Updated `throughput.ts`, `latency.ts`, and `user_touches.ts` to use the GitHub Search API for items merged/closed in the last 7 days. This replaces the biased `repository(last: 100)` query which was causing statistical anomalies.
- **Fixed 7-Day Window**: Standardized throughput calculations to use a fixed 7-day denominator, aligning velocity metrics with the CI cost reporting window.
- **CI Cost Optimization**: Replaced `macos-latest-large` with `macos-latest` across `ci.yml`, `chained_e2e.yml`, and `deflake.yml`.
- **Matrix Reduction**: Reduced the `test_mac` matrix in `ci.yml` to Node 20.x only, significantly cutting down on redundant Mac CI minutes.

### Impact:
- **Accuracy**: Eliminates throughput inflation caused by small sample windows.
- **Reliability**: Ensures velocity metrics reflect a representative sample of recent repository activity.
- **Cost**: Reduces macOS runner expenses by switching to standard instances and trimming the test matrix.

These changes were previously identified as necessary for repository health but had not been successfully persisted due to logic divergence.
2026-05-12 16:53:13 +00:00

507 lines
18 KiB
YAML

name: 'Testing: CI'
on:
push:
branches:
- 'main'
- 'release/**'
pull_request:
branches:
- 'main'
- 'release/**'
merge_group:
workflow_dispatch:
inputs:
branch_ref:
description: 'Branch to run on'
required: true
default: 'main'
type: 'string'
concurrency:
group: '${{ github.workflow }}-${{ github.head_ref || github.ref }}'
cancel-in-progress: |-
${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/heads/release/') }}
permissions:
checks: 'write'
contents: 'read'
statuses: 'write'
defaults:
run:
shell: 'bash'
jobs:
merge_queue_skipper:
permissions: 'read-all'
name: 'Merge Queue Skipper'
runs-on: 'gemini-cli-ubuntu-16-core'
if: "github.repository == 'google-gemini/gemini-cli'"
outputs:
skip: '${{ steps.merge-queue-ci-skipper.outputs.skip-check }}'
steps:
- id: 'merge-queue-ci-skipper'
uses: 'cariad-tech/merge-queue-ci-skipper@1032489e59437862c90a08a2c92809c903883772' # ratchet:cariad-tech/merge-queue-ci-skipper@main
with:
secret: '${{ secrets.GEMINI_CLI_ROBOT_GITHUB_PAT }}'
lint:
name: 'Lint'
runs-on: 'gemini-cli-ubuntu-16-core'
needs: 'merge_queue_skipper'
if: "github.repository == 'google-gemini/gemini-cli' && needs.merge_queue_skipper.outputs.skip == 'false'"
env:
GEMINI_LINT_TEMP_DIR: '${{ github.workspace }}/.gemini-linters'
steps:
- name: 'Checkout'
uses: 'actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8' # ratchet:actions/checkout@v5
with:
ref: '${{ github.event.inputs.branch_ref || github.ref }}'
fetch-depth: 0
- name: 'Set up Node.js'
uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions/setup-node@v4.4.0
with:
node-version-file: '.nvmrc'
cache: 'npm'
- name: 'Cache Linters'
uses: 'actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830' # ratchet:actions/cache@v4
with:
path: '${{ env.GEMINI_LINT_TEMP_DIR }}'
key: "${{ runner.os }}-${{ runner.arch }}-linters-${{ hashFiles('scripts/lint.js') }}"
- name: 'Install dependencies'
run: 'npm ci'
- name: 'Cache ESLint'
uses: 'actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830' # ratchet:actions/cache@v4
with:
path: '.eslintcache'
key: "${{ runner.os }}-eslint-${{ hashFiles('package-lock.json', 'eslint.config.js') }}"
- name: 'Validate NOTICES.txt'
run: 'git diff --exit-code packages/vscode-ide-companion/NOTICES.txt'
- name: 'Check lockfile'
run: 'npm run check:lockfile'
- name: 'Install linters'
run: 'node scripts/lint.js --setup'
- name: 'Run ESLint'
run: 'node scripts/lint.js --eslint'
- name: 'Run actionlint'
run: 'node scripts/lint.js --actionlint'
- name: 'Run shellcheck'
run: 'node scripts/lint.js --shellcheck'
- name: 'Run yamllint'
run: 'node scripts/lint.js --yamllint'
- name: 'Build project for typecheck'
run: 'npm run build'
- name: 'Run typecheck'
run: 'npm run typecheck'
- name: 'Run Prettier'
run: 'node scripts/lint.js --prettier'
- name: 'Build docs prerequisites'
run: 'npm run predocs:settings'
- name: 'Verify settings docs'
run: 'npm run docs:settings -- --check'
- name: 'Run sensitive keyword linter'
run: 'node scripts/lint.js --sensitive-keywords'
- name: 'Run GitHub Actions pinning linter'
run: 'node scripts/lint.js --check-github-actions-pinning'
link_checker:
name: 'Link Checker'
runs-on: 'ubuntu-latest'
if: "github.repository == 'google-gemini/gemini-cli'"
steps:
- name: 'Checkout'
uses: 'actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8' # ratchet:actions/checkout@v5
- name: 'Link Checker'
uses: 'lycheeverse/lychee-action@885c65f3dc543b57c898c8099f4e08c8afd178a2' # ratchet: lycheeverse/lychee-action@v2.6.1
with:
args: '--verbose --accept 200,503 ./**/*.md'
fail: true
test_linux:
name: 'Test (Linux) - ${{ matrix.node-version }}, ${{ matrix.shard }}'
runs-on: 'gemini-cli-ubuntu-16-core'
needs:
- 'merge_queue_skipper'
if: "github.repository == 'google-gemini/gemini-cli' && needs.merge_queue_skipper.outputs.skip == 'false'"
permissions:
contents: 'read'
checks: 'write'
pull-requests: 'write'
strategy:
matrix:
node-version:
- '20.x'
- '22.x'
- '24.x'
shard:
- 'cli'
- 'others'
steps:
- name: 'Checkout'
uses: 'actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8' # ratchet:actions/checkout@v5
- name: 'Set up Node.js ${{ matrix.node-version }}'
uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions/setup-node@v4
with:
node-version: '${{ matrix.node-version }}'
cache: 'npm'
- name: 'Build project'
run: 'npm run build'
- name: 'Install system dependencies'
run: |
sudo apt-get update -qq && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq bubblewrap
# Ubuntu 24.04+ requires this to allow bwrap to function in CI
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true
- name: 'Install dependencies for testing'
run: 'npm ci'
- name: 'Run tests and generate reports'
env:
NO_COLOR: true
GEMINI_CLI_TRUST_WORKSPACE: true
run: |
if [[ "${{ matrix.shard }}" == "cli" ]]; then
npm run test:ci --workspace "@google/gemini-cli"
else
# Explicitly list non-cli packages to ensure they are sharded correctly
npm run test:ci --workspace "@google/gemini-cli-core" --workspace "@google/gemini-cli-a2a-server" --workspace "gemini-cli-vscode-ide-companion" --workspace "@google/gemini-cli-test-utils" --if-present -- --coverage.enabled=false
npm run test:scripts
fi
- name: 'Bundle'
run: 'npm run bundle'
- name: 'Smoke test bundle'
run: 'node ./bundle/gemini.js --version'
- name: 'Smoke test npx installation'
run: |
# 1. Package the project into a tarball
TARBALL=$(npm pack | tail -n 1)
# 2. Move to a fresh directory for isolation
mkdir -p ../smoke-test-dir
mv "$TARBALL" ../smoke-test-dir/
cd ../smoke-test-dir
# 3. Run npx from the tarball
npx "./$TARBALL" --version
- name: 'Wait for file system sync'
run: 'sleep 2'
- name: 'Publish Test Report (for non-forks)'
if: |-
${{ always() && (github.event.pull_request.head.repo.full_name == github.repository) }}
uses: 'dorny/test-reporter@dc3a92680fcc15842eef52e8c4606ea7ce6bd3f3' # ratchet:dorny/test-reporter@v2
with:
name: 'Test Results (Node ${{ runner.os }}, ${{ matrix.node-version }}, ${{ matrix.shard }})'
path: 'packages/*/junit.xml'
reporter: 'java-junit'
fail-on-error: 'false'
- name: 'Upload Test Results Artifact (for forks)'
if: |-
${{ always() && (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) }}
uses: 'actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02' # ratchet:actions/upload-artifact@v4
with:
name: 'test-results-fork-${{ runner.os }}-${{ matrix.node-version }}-${{ matrix.shard }}'
path: 'packages/*/junit.xml'
test_mac:
name: 'Test (Mac) - ${{ matrix.node-version }}, ${{ matrix.shard }}'
runs-on: 'macos-latest'
needs:
- 'merge_queue_skipper'
if: "github.repository == 'google-gemini/gemini-cli' && needs.merge_queue_skipper.outputs.skip == 'false'"
permissions:
contents: 'read'
checks: 'write'
pull-requests: 'write'
continue-on-error: true
strategy:
matrix:
node-version:
- '20.x'
shard:
- 'cli'
- 'others'
steps:
- name: 'Checkout'
uses: 'actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8' # ratchet:actions/checkout@v5
- name: 'Set up Node.js ${{ matrix.node-version }}'
uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions/setup-node@v4
with:
node-version: '${{ matrix.node-version }}'
cache: 'npm'
- name: 'Build project'
run: 'npm run build'
- name: 'Install dependencies for testing'
run: 'npm ci'
- name: 'Run tests and generate reports'
env:
NO_COLOR: true
GEMINI_CLI_TRUST_WORKSPACE: true
run: |
if [[ "${{ matrix.shard }}" == "cli" ]]; then
npm run test:ci --workspace "@google/gemini-cli" -- --coverage.enabled=false
else
# Explicitly list non-cli packages to ensure they are sharded correctly
npm run test:ci --workspace "@google/gemini-cli-core" --workspace "@google/gemini-cli-a2a-server" --workspace "gemini-cli-vscode-ide-companion" --workspace "@google/gemini-cli-test-utils" --if-present -- --coverage.enabled=false
npm run test:scripts
fi
- name: 'Bundle'
run: 'npm run bundle'
- name: 'Smoke test bundle'
run: 'node ./bundle/gemini.js --version'
- name: 'Smoke test npx installation'
run: |
# 1. Package the project into a tarball
TARBALL=$(npm pack | tail -n 1)
# 2. Move to a fresh directory for isolation
mkdir -p ../smoke-test-dir
mv "$TARBALL" ../smoke-test-dir/
cd ../smoke-test-dir
# 3. Run npx from the tarball
npx "./$TARBALL" --version
- name: 'Wait for file system sync'
run: 'sleep 2'
- name: 'Publish Test Report (for non-forks)'
if: |-
${{ always() && (github.event.pull_request.head.repo.full_name == github.repository) }}
uses: 'dorny/test-reporter@dc3a92680fcc15842eef52e8c4606ea7ce6bd3f3' # ratchet:dorny/test-reporter@v2
with:
name: 'Test Results (Node ${{ runner.os }}, ${{ matrix.node-version }}, ${{ matrix.shard }})'
path: 'packages/*/junit.xml'
reporter: 'java-junit'
fail-on-error: 'false'
- name: 'Upload Test Results Artifact (for forks)'
if: |-
${{ always() && (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) }}
uses: 'actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02' # ratchet:actions/upload-artifact@v4
with:
name: 'test-results-fork-${{ runner.os }}-${{ matrix.node-version }}-${{ matrix.shard }}'
path: 'packages/*/junit.xml'
- name: 'Upload coverage reports'
if: |-
${{ always() }}
uses: 'actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02' # ratchet:actions/upload-artifact@v4
with:
name: 'coverage-reports-${{ runner.os }}-${{ matrix.node-version }}-${{ matrix.shard }}'
path: 'packages/*/coverage'
codeql:
name: 'CodeQL'
runs-on: 'gemini-cli-ubuntu-16-core'
needs: 'merge_queue_skipper'
if: "github.repository == 'google-gemini/gemini-cli' && needs.merge_queue_skipper.outputs.skip == 'false'"
permissions:
actions: 'read'
contents: 'read'
security-events: 'write'
steps:
- name: 'Checkout'
uses: 'actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8' # ratchet:actions/checkout@v5
with:
ref: '${{ github.event.inputs.branch_ref || github.ref }}'
- name: 'Initialize CodeQL'
uses: 'github/codeql-action/init@df559355d593797519d70b90fc8edd5db049e7a2' # ratchet:github/codeql-action/init@v3
with:
languages: 'javascript'
- name: 'Perform CodeQL Analysis'
uses: 'github/codeql-action/analyze@df559355d593797519d70b90fc8edd5db049e7a2' # ratchet:github/codeql-action/analyze@v3
# Check for changes in bundle size.
bundle_size:
name: 'Check Bundle Size'
needs: 'merge_queue_skipper'
if: "github.repository == 'google-gemini/gemini-cli' && github.event_name == 'pull_request' && needs.merge_queue_skipper.outputs.skip == 'false'"
runs-on: 'gemini-cli-ubuntu-16-core'
permissions:
contents: 'read' # For checkout
pull-requests: 'write' # For commenting
steps:
- name: 'Checkout'
uses: 'actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8' # ratchet:actions/checkout@v5
with:
ref: '${{ github.event.inputs.branch_ref || github.ref }}'
fetch-depth: 1
- uses: 'preactjs/compressed-size-action@946a292cd35bd1088e0d7eb92b69d1a8d5b5d76a'
with:
repo-token: '${{ secrets.GITHUB_TOKEN }}'
pattern: './bundle/**/*.{js,sb}'
minimum-change-threshold: '1000'
compression: 'none'
clean-script: 'clean'
test_windows:
name: 'Slow Test - Win - ${{ matrix.shard }}'
runs-on: 'gemini-cli-windows-16-core'
needs: 'merge_queue_skipper'
if: "github.repository == 'google-gemini/gemini-cli' && needs.merge_queue_skipper.outputs.skip == 'false'"
timeout-minutes: 60
strategy:
matrix:
shard:
- 'cli'
- 'others'
steps:
- name: 'Checkout'
uses: 'actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8' # ratchet:actions/checkout@v5
with:
ref: '${{ github.event.inputs.branch_ref || github.ref }}'
- name: 'Set up Node.js 20.x'
uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions/setup-node@v4
with:
node-version: '20.x'
cache: 'npm'
- name: 'Configure Windows Defender exclusions'
run: |
Add-MpPreference -ExclusionPath $env:GITHUB_WORKSPACE -Force
Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE\node_modules" -Force
Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE\packages" -Force
Add-MpPreference -ExclusionPath "$env:TEMP" -Force
shell: 'pwsh'
- name: 'Configure npm for Windows performance'
run: |
npm config set progress false
npm config set audit false
npm config set fund false
npm config set loglevel error
npm config set maxsockets 32
npm config set registry https://registry.npmjs.org/
shell: 'pwsh'
- name: 'Install dependencies'
run: 'npm ci'
shell: 'pwsh'
- name: 'Build project'
run: 'npm run build'
shell: 'pwsh'
env:
NODE_OPTIONS: '--max-old-space-size=32768 --max-semi-space-size=256'
UV_THREADPOOL_SIZE: '32'
NODE_ENV: 'production'
- name: 'Run tests and generate reports'
env:
GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
NO_COLOR: true
GEMINI_CLI_TRUST_WORKSPACE: true
NODE_OPTIONS: '--max-old-space-size=32768 --max-semi-space-size=256'
UV_THREADPOOL_SIZE: '32'
NODE_ENV: 'test'
run: |
if ("${{ matrix.shard }}" -eq "cli") {
npm run test:ci --workspace "@google/gemini-cli" -- --coverage.enabled=false
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
} else {
# Explicitly list non-cli packages to ensure they are sharded correctly
npm run test:ci --workspace "@google/gemini-cli-core" --workspace "@google/gemini-cli-a2a-server" --workspace "gemini-cli-vscode-ide-companion" --workspace "@google/gemini-cli-test-utils" --if-present -- --coverage.enabled=false
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
npm run test:scripts
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
}
shell: 'pwsh'
- name: 'Bundle'
run: 'npm run bundle'
shell: 'pwsh'
- name: 'Smoke test bundle'
run: 'node ./bundle/gemini.js --version'
shell: 'pwsh'
- name: 'Smoke test npx installation'
run: |
# 1. Package the project into a tarball
$PACK_OUTPUT = npm pack
$TARBALL = $PACK_OUTPUT[-1]
# 2. Move to a fresh directory for isolation
New-Item -ItemType Directory -Force -Path ../smoke-test-dir
Move-Item $TARBALL ../smoke-test-dir/
Set-Location ../smoke-test-dir
# 3. Run npx from the tarball
npx "./$TARBALL" --version
shell: 'pwsh'
ci:
name: 'CI'
if: "github.repository == 'google-gemini/gemini-cli' && always()"
needs:
- 'lint'
- 'link_checker'
- 'test_linux'
- 'test_mac'
- 'test_windows'
- 'codeql'
- 'bundle_size'
runs-on: 'gemini-cli-ubuntu-16-core'
steps:
- name: 'Check all job results'
run: |
if [[ (${NEEDS_LINT_RESULT} != 'success' && ${NEEDS_LINT_RESULT} != 'skipped') || \
(${NEEDS_LINK_CHECKER_RESULT} != 'success' && ${NEEDS_LINK_CHECKER_RESULT} != 'skipped') || \
(${NEEDS_TEST_LINUX_RESULT} != 'success' && ${NEEDS_TEST_LINUX_RESULT} != 'skipped') || \
(${NEEDS_TEST_MAC_RESULT} != 'success' && ${NEEDS_TEST_MAC_RESULT} != 'skipped') || \
(${NEEDS_TEST_WINDOWS_RESULT} != 'success' && ${NEEDS_TEST_WINDOWS_RESULT} != 'skipped') || \
(${NEEDS_CODEQL_RESULT} != 'success' && ${NEEDS_CODEQL_RESULT} != 'skipped') || \
(${NEEDS_BUNDLE_SIZE_RESULT} != 'success' && ${NEEDS_BUNDLE_SIZE_RESULT} != 'skipped') ]]; then
echo "One or more CI jobs failed."
exit 1
fi
echo "All CI jobs passed!"
env:
NEEDS_LINT_RESULT: '${{ needs.lint.result }}'
NEEDS_LINK_CHECKER_RESULT: '${{ needs.link_checker.result }}'
NEEDS_TEST_LINUX_RESULT: '${{ needs.test_linux.result }}'
NEEDS_TEST_MAC_RESULT: '${{ needs.test_mac.result }}'
NEEDS_TEST_WINDOWS_RESULT: '${{ needs.test_windows.result }}'
NEEDS_CODEQL_RESULT: '${{ needs.codeql.result }}'
NEEDS_BUNDLE_SIZE_RESULT: '${{ needs.bundle_size.result }}'