mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-05-24 02:33:18 -07:00
gemini-cli[bot]
17aec810ca
This PR restores critical fixes for repository metrics and CI cost optimizations that were identified as missing from the filesystem despite being marked as completed in previous tasks. ### Changes: - **Metrics Accuracy**: Re-implemented 7-day fixed window and search-based sampling in `throughput.ts`, `latency.ts`, and `user_touches.ts` to resolve reporting anomalies during batch operations. - **Spend Tracking**: Implemented pagination in `actions_spend.ts` to ensure all workflow runs within the 7-day window are captured, avoiding undercounting. - **CI Cost Optimization**: Replaced all instances of `macos-latest-large` with standard `macos-latest` runners in `ci.yml`, `chained_e2e.yml`, and `deflake.yml`. - **Matrix Reduction**: Reduced the Mac test matrix in `ci.yml` to Node 20.x only, significantly reducing redundant compute spend. - **Task Ledger**: Updated `lessons-learned.md` to document the logic divergence and its resolution (BT-63). These changes ensure the repository metrics are reliable and that CI costs remain under control.
403 lines
15 KiB
YAML
403 lines
15 KiB
YAML
name: 'Testing: E2E (Chained)'
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- 'main'
|
|
merge_group:
|
|
workflow_run:
|
|
workflows: ['Trigger E2E']
|
|
types: ['completed']
|
|
workflow_dispatch:
|
|
inputs:
|
|
head_sha:
|
|
description: 'SHA of the commit to test'
|
|
required: true
|
|
repo_name:
|
|
description: 'Repository name (e.g., owner/repo)'
|
|
required: true
|
|
|
|
concurrency:
|
|
group: '${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.ref }}'
|
|
cancel-in-progress: |-
|
|
${{ github.event_name != 'push' && github.event_name != 'merge_group' }}
|
|
|
|
permissions:
|
|
contents: 'read'
|
|
statuses: 'write'
|
|
|
|
jobs:
|
|
merge_queue_skipper:
|
|
name: 'Merge Queue Skipper'
|
|
permissions: 'read-all'
|
|
runs-on: 'gemini-cli-ubuntu-16-core'
|
|
if: "github.repository == 'google-gemini/gemini-cli'"
|
|
outputs:
|
|
skip: '${{ steps.merge-queue-e2e-skipper.outputs.skip-check }}'
|
|
steps:
|
|
- id: 'merge-queue-e2e-skipper'
|
|
uses: 'cariad-tech/merge-queue-ci-skipper@1032489e59437862c90a08a2c92809c903883772' # ratchet:cariad-tech/merge-queue-ci-skipper@main
|
|
with:
|
|
secret: '${{ secrets.GEMINI_CLI_ROBOT_GITHUB_PAT }}'
|
|
continue-on-error: true
|
|
|
|
download_repo_name:
|
|
runs-on: 'gemini-cli-ubuntu-16-core'
|
|
if: "github.repository == 'google-gemini/gemini-cli' && (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run')"
|
|
outputs:
|
|
repo_name: '${{ steps.output-repo-name.outputs.repo_name }}'
|
|
head_sha: '${{ steps.output-repo-name.outputs.head_sha }}'
|
|
steps:
|
|
- name: 'Mock Repo Artifact'
|
|
if: "${{ github.event_name == 'workflow_dispatch' }}"
|
|
env:
|
|
REPO_NAME: '${{ github.event.inputs.repo_name }}'
|
|
run: |
|
|
mkdir -p ./pr
|
|
echo "${REPO_NAME}" > ./pr/repo_name
|
|
- uses: 'actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02' # ratchet:actions/upload-artifact@v4
|
|
with:
|
|
name: 'repo_name'
|
|
path: 'pr/'
|
|
- name: 'Download the repo_name artifact'
|
|
uses: 'actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0' # ratchet:actions/download-artifact@v5
|
|
env:
|
|
RUN_ID: "${{ github.event_name == 'workflow_run' && github.event.workflow_run.id || github.run_id }}"
|
|
with:
|
|
github-token: '${{ secrets.GITHUB_TOKEN }}'
|
|
name: 'repo_name'
|
|
run-id: '${{ env.RUN_ID }}'
|
|
path: '${{ runner.temp }}/artifacts'
|
|
- name: 'Output Repo Name and SHA'
|
|
id: 'output-repo-name'
|
|
uses: 'actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd' # ratchet:actions/github-script@v8
|
|
with:
|
|
github-token: '${{ secrets.GITHUB_TOKEN }}'
|
|
script: |
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const temp = '${{ runner.temp }}/artifacts';
|
|
const repoPath = path.join(temp, 'repo_name');
|
|
if (fs.existsSync(repoPath)) {
|
|
const repo_name = String(fs.readFileSync(repoPath)).trim();
|
|
core.setOutput('repo_name', repo_name);
|
|
}
|
|
const shaPath = path.join(temp, 'head_sha');
|
|
if (fs.existsSync(shaPath)) {
|
|
const head_sha = String(fs.readFileSync(shaPath)).trim();
|
|
core.setOutput('head_sha', head_sha);
|
|
}
|
|
|
|
parse_run_context:
|
|
name: 'Parse run context'
|
|
runs-on: 'gemini-cli-ubuntu-16-core'
|
|
needs: 'download_repo_name'
|
|
if: "github.repository == 'google-gemini/gemini-cli' && always()"
|
|
outputs:
|
|
repository: '${{ steps.set_context.outputs.REPO }}'
|
|
sha: '${{ steps.set_context.outputs.SHA }}'
|
|
steps:
|
|
- id: 'set_context'
|
|
name: 'Set dynamic repository and SHA'
|
|
env:
|
|
REPO: '${{ needs.download_repo_name.outputs.repo_name || github.repository }}'
|
|
SHA: '${{ needs.download_repo_name.outputs.head_sha || github.event.inputs.head_sha || github.event.workflow_run.head_sha || github.sha }}'
|
|
shell: 'bash'
|
|
run: |
|
|
echo "REPO=$REPO" >> "$GITHUB_OUTPUT"
|
|
echo "SHA=$SHA" >> "$GITHUB_OUTPUT"
|
|
|
|
set_pending_status:
|
|
runs-on: 'gemini-cli-ubuntu-16-core'
|
|
permissions: 'write-all'
|
|
needs:
|
|
- 'parse_run_context'
|
|
if: "github.repository == 'google-gemini/gemini-cli' && always()"
|
|
steps:
|
|
- name: 'Set pending status'
|
|
uses: 'myrotvorets/set-commit-status-action@16037e056d73b2d3c88e37e393ff369047f70886' # ratchet:myrotvorets/set-commit-status-action@master
|
|
if: "github.repository == 'google-gemini/gemini-cli' && always()"
|
|
with:
|
|
allowForks: 'true'
|
|
repo: '${{ github.repository }}'
|
|
sha: '${{ needs.parse_run_context.outputs.sha }}'
|
|
token: '${{ secrets.GEMINI_CLI_ROBOT_GITHUB_PAT }}'
|
|
status: 'pending'
|
|
context: 'E2E (Chained)'
|
|
|
|
e2e_linux:
|
|
name: 'E2E Test (Linux) - ${{ matrix.sandbox }}'
|
|
needs:
|
|
- 'merge_queue_skipper'
|
|
- 'parse_run_context'
|
|
runs-on: 'gemini-cli-ubuntu-16-core'
|
|
if: |
|
|
github.repository == 'google-gemini/gemini-cli' && always() && (needs.merge_queue_skipper.result !='success' || needs.merge_queue_skipper.outputs.skip != 'true')
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
sandbox:
|
|
- 'sandbox:none'
|
|
- 'sandbox:docker'
|
|
node-version:
|
|
- '20.x'
|
|
|
|
steps:
|
|
- name: 'Checkout'
|
|
uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5
|
|
with:
|
|
ref: '${{ needs.parse_run_context.outputs.sha }}'
|
|
repository: '${{ needs.parse_run_context.outputs.repository }}'
|
|
|
|
- name: 'Set up Node.js ${{ matrix.node-version }}'
|
|
uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4
|
|
with:
|
|
node-version: '${{ matrix.node-version }}'
|
|
|
|
- name: 'Install dependencies'
|
|
run: 'npm ci'
|
|
|
|
- name: 'Build project'
|
|
run: 'npm run build'
|
|
|
|
- name: 'Set up Docker'
|
|
if: "${{matrix.sandbox == 'sandbox:docker'}}"
|
|
uses: 'docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435' # ratchet:docker/setup-buildx-action@v3
|
|
|
|
- name: 'Run E2E tests'
|
|
env:
|
|
GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
|
|
GEMINI_CLI_TRUST_WORKSPACE: true
|
|
KEEP_OUTPUT: 'true'
|
|
VERBOSE: 'true'
|
|
BUILD_SANDBOX_FLAGS: '--cache-from type=gha --cache-to type=gha,mode=max'
|
|
shell: 'bash'
|
|
run: |
|
|
if [[ "${{ matrix.sandbox }}" == "sandbox:docker" ]]; then
|
|
npm run test:integration:sandbox:docker
|
|
else
|
|
npm run test:integration:sandbox:none
|
|
fi
|
|
|
|
e2e_mac:
|
|
name: 'E2E Test (macOS)'
|
|
needs:
|
|
- 'merge_queue_skipper'
|
|
- 'parse_run_context'
|
|
runs-on: 'macos-latest'
|
|
if: |
|
|
github.repository == 'google-gemini/gemini-cli' && always() && (needs.merge_queue_skipper.result !='success' || needs.merge_queue_skipper.outputs.skip != 'true')
|
|
steps:
|
|
- name: 'Checkout'
|
|
uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5
|
|
with:
|
|
ref: '${{ needs.parse_run_context.outputs.sha }}'
|
|
repository: '${{ needs.parse_run_context.outputs.repository }}'
|
|
|
|
- name: 'Set up Node.js 20.x'
|
|
uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4
|
|
with:
|
|
node-version: '20.x'
|
|
|
|
- name: 'Install dependencies'
|
|
run: 'npm ci'
|
|
|
|
- name: 'Build project'
|
|
run: 'npm run build'
|
|
|
|
- name: 'Fix rollup optional dependencies on macOS'
|
|
if: "${{runner.os == 'macOS'}}"
|
|
run: |
|
|
npm cache clean --force
|
|
- name: 'Run E2E tests (non-Windows)'
|
|
if: "${{runner.os != 'Windows'}}"
|
|
env:
|
|
GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
|
|
GEMINI_CLI_TRUST_WORKSPACE: true
|
|
KEEP_OUTPUT: 'true'
|
|
SANDBOX: 'sandbox:none'
|
|
VERBOSE: 'true'
|
|
run: 'npm run test:integration:sandbox:none'
|
|
|
|
e2e_windows:
|
|
name: 'Slow E2E - Win'
|
|
needs:
|
|
- 'merge_queue_skipper'
|
|
- 'parse_run_context'
|
|
if: |
|
|
github.repository == 'google-gemini/gemini-cli' && always() && (needs.merge_queue_skipper.result !='success' || needs.merge_queue_skipper.outputs.skip != 'true')
|
|
runs-on: 'gemini-cli-windows-16-core'
|
|
steps:
|
|
- name: 'Checkout'
|
|
uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5
|
|
with:
|
|
ref: '${{ needs.parse_run_context.outputs.sha }}'
|
|
repository: '${{ needs.parse_run_context.outputs.repository }}'
|
|
|
|
- name: 'Set up Node.js 20.x'
|
|
uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4
|
|
with:
|
|
node-version: '20.x'
|
|
cache: 'npm'
|
|
|
|
- name: 'Configure Windows Defender exclusions'
|
|
run: |
|
|
Add-MpPreference -ExclusionPath $env:GITHUB_WORKSPACE -Force
|
|
Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE\node_modules" -Force
|
|
Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE\packages" -Force
|
|
Add-MpPreference -ExclusionPath "$env:TEMP" -Force
|
|
shell: 'pwsh'
|
|
|
|
- name: 'Configure npm for Windows performance'
|
|
run: |
|
|
npm config set progress false
|
|
npm config set audit false
|
|
npm config set fund false
|
|
npm config set loglevel error
|
|
npm config set maxsockets 32
|
|
npm config set registry https://registry.npmjs.org/
|
|
shell: 'pwsh'
|
|
|
|
- name: 'Install dependencies'
|
|
run: 'npm ci'
|
|
shell: 'pwsh'
|
|
|
|
- name: 'Build project'
|
|
run: 'npm run build'
|
|
shell: 'pwsh'
|
|
|
|
- name: 'Ensure Chrome is available'
|
|
shell: 'pwsh'
|
|
run: |
|
|
$chromePaths = @(
|
|
"${env:ProgramFiles}\Google\Chrome\Application\chrome.exe",
|
|
"${env:ProgramFiles(x86)}\Google\Chrome\Application\chrome.exe"
|
|
)
|
|
$chromeExists = $chromePaths | Where-Object { Test-Path $_ } | Select-Object -First 1
|
|
if (-not $chromeExists) {
|
|
Write-Host 'Chrome not found, installing via Chocolatey...'
|
|
choco install googlechrome -y --no-progress --ignore-checksums
|
|
}
|
|
$installed = $chromePaths | Where-Object { Test-Path $_ } | Select-Object -First 1
|
|
if ($installed) {
|
|
Write-Host "Chrome found at: $installed"
|
|
& $installed --version
|
|
} else {
|
|
Write-Error 'Chrome installation failed'
|
|
exit 1
|
|
}
|
|
|
|
- name: 'Run E2E tests'
|
|
env:
|
|
GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
|
|
GEMINI_CLI_TRUST_WORKSPACE: true
|
|
KEEP_OUTPUT: 'true'
|
|
SANDBOX: 'sandbox:none'
|
|
VERBOSE: 'true'
|
|
NODE_OPTIONS: '--max-old-space-size=32768 --max-semi-space-size=256'
|
|
UV_THREADPOOL_SIZE: '32'
|
|
NODE_ENV: 'test'
|
|
shell: 'pwsh'
|
|
run: 'npm run test:integration:sandbox:none'
|
|
|
|
evals:
|
|
name: 'Evals (ALWAYS_PASSING)'
|
|
needs:
|
|
- 'merge_queue_skipper'
|
|
- 'parse_run_context'
|
|
runs-on: 'gemini-cli-ubuntu-16-core'
|
|
if: |
|
|
github.repository == 'google-gemini/gemini-cli' && always() && (needs.merge_queue_skipper.result !='success' || needs.merge_queue_skipper.outputs.skip != 'true')
|
|
steps:
|
|
- name: 'Checkout'
|
|
uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5
|
|
with:
|
|
ref: '${{ needs.parse_run_context.outputs.sha }}'
|
|
repository: '${{ needs.parse_run_context.outputs.repository }}'
|
|
fetch-depth: 0
|
|
|
|
- name: 'Set up Node.js 20.x'
|
|
uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4
|
|
with:
|
|
node-version: '20.x'
|
|
|
|
- name: 'Install dependencies'
|
|
run: 'npm ci'
|
|
|
|
- name: 'Build project'
|
|
run: 'npm run build'
|
|
|
|
- name: 'Check if evals should run'
|
|
id: 'check_evals'
|
|
run: |
|
|
SHOULD_RUN=$(node scripts/changed_prompt.js)
|
|
echo "should_run=$SHOULD_RUN" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: 'Run Evals (Required to pass)'
|
|
if: "${{ steps.check_evals.outputs.should_run == 'true' }}"
|
|
env:
|
|
GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
|
|
GEMINI_CLI_TRUST_WORKSPACE: true
|
|
GEMINI_MODEL: 'gemini-3-pro-preview'
|
|
# Only run always passes behavioral tests.
|
|
EVAL_SUITE_TYPE: 'behavioral'
|
|
# Disable Vitest internal retries to avoid double-retrying;
|
|
# custom retry logic is handled in evals/test-helper.ts
|
|
VITEST_RETRY: 0
|
|
run: 'npm run test:always_passing_evals'
|
|
|
|
- name: 'Upload Reliability Logs'
|
|
if: "always() && steps.check_evals.outputs.should_run == 'true'"
|
|
uses: 'actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02' # ratchet:actions/upload-artifact@v4
|
|
with:
|
|
name: 'eval-logs-${{ github.run_id }}-${{ github.run_attempt }}'
|
|
path: 'evals/logs/api-reliability.jsonl'
|
|
retention-days: 7
|
|
|
|
e2e:
|
|
name: 'E2E'
|
|
if: |
|
|
github.repository == 'google-gemini/gemini-cli' && always() && (needs.merge_queue_skipper.result !='success' || needs.merge_queue_skipper.outputs.skip != 'true')
|
|
needs:
|
|
- 'e2e_linux'
|
|
- 'e2e_mac'
|
|
- 'e2e_windows'
|
|
- 'evals'
|
|
- 'merge_queue_skipper'
|
|
runs-on: 'gemini-cli-ubuntu-16-core'
|
|
steps:
|
|
- name: 'Check E2E test results'
|
|
run: |
|
|
if [[ ${NEEDS_E2E_LINUX_RESULT} != 'success' || \
|
|
${NEEDS_E2E_MAC_RESULT} != 'success' || \
|
|
${NEEDS_E2E_WINDOWS_RESULT} != 'success' || \
|
|
${NEEDS_EVALS_RESULT} != 'success' ]]; then
|
|
echo "One or more E2E jobs failed."
|
|
exit 1
|
|
fi
|
|
echo "All required E2E jobs passed!"
|
|
env:
|
|
NEEDS_E2E_LINUX_RESULT: '${{ needs.e2e_linux.result }}'
|
|
NEEDS_E2E_MAC_RESULT: '${{ needs.e2e_mac.result }}'
|
|
NEEDS_E2E_WINDOWS_RESULT: '${{ needs.e2e_windows.result }}'
|
|
NEEDS_EVALS_RESULT: '${{ needs.evals.result }}'
|
|
|
|
set_workflow_status:
|
|
runs-on: 'gemini-cli-ubuntu-16-core'
|
|
permissions: 'write-all'
|
|
if: "github.repository == 'google-gemini/gemini-cli' && always()"
|
|
needs:
|
|
- 'parse_run_context'
|
|
- 'e2e'
|
|
steps:
|
|
- name: 'Set workflow status'
|
|
uses: 'myrotvorets/set-commit-status-action@16037e056d73b2d3c88e37e393ff369047f70886' # ratchet:myrotvorets/set-commit-status-action@master
|
|
if: "github.repository == 'google-gemini/gemini-cli' && always()"
|
|
with:
|
|
allowForks: 'true'
|
|
repo: '${{ github.repository }}'
|
|
sha: '${{ needs.parse_run_context.outputs.sha }}'
|
|
token: '${{ secrets.GITHUB_TOKEN }}'
|
|
status: '${{ needs.e2e.result }}'
|
|
context: 'E2E (Chained)'
|