mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-07-10 01:50:20 -07:00
Merge remote-tracking branch 'origin/main' into component-eval
This commit is contained in:
Christian Gunderman
@@ -175,10 +175,10 @@ jobs:
|
||||
NO_COLOR: true
|
||||
run: |
|
||||
if [[ "${{ matrix.shard }}" == "cli" ]]; then
|
||||
npm run test:ci --workspace @google/gemini-cli
|
||||
npm run test:ci --workspace "@google/gemini-cli"
|
||||
else
|
||||
# Explicitly list non-cli packages to ensure they are sharded correctly
|
||||
npm run test:ci --workspace @google/gemini-cli-core --workspace @google/gemini-cli-a2a-server --workspace gemini-cli-vscode-ide-companion --workspace @google/gemini-cli-test-utils --if-present -- --coverage.enabled=false
|
||||
npm run test:ci --workspace "@google/gemini-cli-core" --workspace "@google/gemini-cli-a2a-server" --workspace "gemini-cli-vscode-ide-companion" --workspace "@google/gemini-cli-test-utils" --if-present -- --coverage.enabled=false
|
||||
npm run test:scripts
|
||||
fi
|
||||
|
||||
@@ -263,10 +263,10 @@ jobs:
|
||||
NO_COLOR: true
|
||||
run: |
|
||||
if [[ "${{ matrix.shard }}" == "cli" ]]; then
|
||||
npm run test:ci --workspace @google/gemini-cli -- --coverage.enabled=false
|
||||
npm run test:ci --workspace "@google/gemini-cli" -- --coverage.enabled=false
|
||||
else
|
||||
# Explicitly list non-cli packages to ensure they are sharded correctly
|
||||
npm run test:ci --workspace @google/gemini-cli-core --workspace @google/gemini-cli-a2a-server --workspace gemini-cli-vscode-ide-companion --workspace @google/gemini-cli-test-utils --if-present -- --coverage.enabled=false
|
||||
npm run test:ci --workspace "@google/gemini-cli-core" --workspace "@google/gemini-cli-a2a-server" --workspace "gemini-cli-vscode-ide-companion" --workspace "@google/gemini-cli-test-utils" --if-present -- --coverage.enabled=false
|
||||
npm run test:scripts
|
||||
fi
|
||||
|
||||
@@ -429,11 +429,14 @@ jobs:
|
||||
NODE_ENV: 'test'
|
||||
run: |
|
||||
if ("${{ matrix.shard }}" -eq "cli") {
|
||||
npm run test:ci --workspace @google/gemini-cli -- --coverage.enabled=false
|
||||
npm run test:ci --workspace "@google/gemini-cli" -- --coverage.enabled=false
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
} else {
|
||||
# Explicitly list non-cli packages to ensure they are sharded correctly
|
||||
npm run test:ci --workspace @google/gemini-cli-core --workspace @google/gemini-cli-a2a-server --workspace gemini-cli-vscode-ide-companion --workspace @google/gemini-cli-test-utils --if-present -- --coverage.enabled=false
|
||||
npm run test:ci --workspace "@google/gemini-cli-core" --workspace "@google/gemini-cli-a2a-server" --workspace "gemini-cli-vscode-ide-companion" --workspace "@google/gemini-cli-test-utils" --if-present -- --coverage.enabled=false
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
npm run test:scripts
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
}
|
||||
shell: 'pwsh'
|
||||
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
name: 'Evals: PR Guidance'
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'packages/core/src/**/*.ts'
|
||||
- '!**/*.test.ts'
|
||||
- '!**/*.test.tsx'
|
||||
|
||||
permissions:
|
||||
pull-requests: 'write'
|
||||
contents: 'read'
|
||||
|
||||
jobs:
|
||||
provide-guidance:
|
||||
name: 'Model Steering Guidance'
|
||||
runs-on: 'ubuntu-latest'
|
||||
if: "github.repository == 'google-gemini/gemini-cli'"
|
||||
steps:
|
||||
- name: 'Checkout'
|
||||
uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: 'Set up Node.js'
|
||||
uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions/setup-node@v4.4.0
|
||||
with:
|
||||
node-version-file: '.nvmrc'
|
||||
cache: 'npm'
|
||||
|
||||
- name: 'Detect Steering Changes'
|
||||
id: 'detect'
|
||||
run: |
|
||||
STEERING_DETECTED=$(node scripts/changed_prompt.js --steering-only)
|
||||
echo "STEERING_DETECTED=$STEERING_DETECTED" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: 'Analyze PR Content'
|
||||
if: "steps.detect.outputs.STEERING_DETECTED == 'true'"
|
||||
id: 'analysis'
|
||||
env:
|
||||
GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
|
||||
run: |
|
||||
# Check for behavioral eval changes
|
||||
EVAL_CHANGES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD | grep "^evals/" || true)
|
||||
if [ -z "$EVAL_CHANGES" ]; then
|
||||
echo "MISSING_EVALS=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Check if user is a maintainer (has write/admin access)
|
||||
USER_PERMISSION=$(gh api repos/${{ github.repository }}/collaborators/${{ github.actor }}/permission --jq '.permission')
|
||||
if [[ "$USER_PERMISSION" == "admin" || "$USER_PERMISSION" == "write" ]]; then
|
||||
echo "IS_MAINTAINER=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: 'Post Guidance Comment'
|
||||
if: "steps.detect.outputs.STEERING_DETECTED == 'true'"
|
||||
uses: 'thollander/actions-comment-pull-request@65f9e5c9a1f2cd378bd74b2e057c9736982a8e74' # ratchet:thollander/actions-comment-pull-request@v3
|
||||
with:
|
||||
comment-tag: 'eval-guidance-bot'
|
||||
message: |
|
||||
### 🧠 Model Steering Guidance
|
||||
|
||||
This PR modifies files that affect the model's behavior (prompts, tools, or instructions).
|
||||
|
||||
${{ steps.analysis.outputs.MISSING_EVALS == 'true' && '- ⚠️ **Consider adding Evals:** No behavioral evaluations (`evals/*.eval.ts`) were added or updated in this PR. Consider adding a test case to verify the new behavior and prevent regressions.' || '' }}
|
||||
${{ steps.analysis.outputs.IS_MAINTAINER == 'true' && '- 🚀 **Maintainer Reminder:** Please ensure that these changes do not regress results on benchmark evals before merging.' || '' }}
|
||||
|
||||
---
|
||||
*This is an automated guidance message triggered by steering logic signatures.*
|
||||
@@ -0,0 +1,209 @@
|
||||
name: 'Evals: PR Evaluation & Regression'
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: ['opened', 'synchronize', 'reopened', 'ready_for_review']
|
||||
paths:
|
||||
- 'packages/core/src/prompts/**'
|
||||
- 'packages/core/src/tools/**'
|
||||
- 'packages/core/src/agents/**'
|
||||
- 'evals/**'
|
||||
- '!**/*.test.ts'
|
||||
- '!**/*.test.tsx'
|
||||
workflow_dispatch:
|
||||
|
||||
# Prevents multiple runs for the same PR simultaneously (saves tokens)
|
||||
concurrency:
|
||||
group: '${{ github.workflow }}-${{ github.head_ref || github.ref }}'
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
pull-requests: 'write'
|
||||
contents: 'read'
|
||||
actions: 'read'
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
name: 'Detect Steering Changes'
|
||||
runs-on: 'gemini-cli-ubuntu-16-core'
|
||||
# Security: pull_request_target allows secrets, so we must gate carefully.
|
||||
# Detection should not run code from the fork.
|
||||
if: "github.repository == 'google-gemini/gemini-cli' && github.event.pull_request.draft == false"
|
||||
outputs:
|
||||
SHOULD_RUN: '${{ steps.detect.outputs.SHOULD_RUN }}'
|
||||
STEERING_DETECTED: '${{ steps.detect.outputs.STEERING_DETECTED }}'
|
||||
steps:
|
||||
- name: 'Checkout'
|
||||
uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5
|
||||
with:
|
||||
# Check out the trusted code from main for detection
|
||||
fetch-depth: 0
|
||||
|
||||
- name: 'Detect Steering Changes'
|
||||
id: 'detect'
|
||||
env:
|
||||
# Use the PR's head SHA for comparison without checking it out
|
||||
PR_HEAD_SHA: '${{ github.event.pull_request.head.sha }}'
|
||||
run: |
|
||||
# Fetch the fork's PR branch for analysis
|
||||
git fetch origin pull/${{ github.event.pull_request.number }}/head:pr-head
|
||||
|
||||
# Run the trusted script from main
|
||||
SHOULD_RUN=$(node scripts/changed_prompt.js)
|
||||
STEERING_DETECTED=$(node scripts/changed_prompt.js --steering-only)
|
||||
echo "SHOULD_RUN=$SHOULD_RUN" >> "$GITHUB_OUTPUT"
|
||||
echo "STEERING_DETECTED=$STEERING_DETECTED" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: 'Notify Approval Required'
|
||||
if: "steps.detect.outputs.SHOULD_RUN == 'true'"
|
||||
env:
|
||||
GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
|
||||
run: |
|
||||
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
COMMENT_BODY="### 🛑 Action Required: Evaluation Approval
|
||||
|
||||
Steering changes have been detected in this PR. To prevent regressions, a maintainer must approve the evaluation run before this PR can be merged.
|
||||
|
||||
**Maintainers:**
|
||||
1. Go to the [**Workflow Run Summary**]($RUN_URL).
|
||||
2. Click the yellow **'Review deployments'** button.
|
||||
3. Select the **'eval-gate'** environment and click **'Approve'**.
|
||||
|
||||
Once approved, the evaluation results will be posted here automatically.
|
||||
|
||||
<!-- eval-approval-notification -->"
|
||||
|
||||
# Check if comment already exists to avoid spamming
|
||||
COMMENT_ID=$(gh pr view ${{ github.event.pull_request.number }} --json comments --jq '.comments[] | select(.body | contains("<!-- eval-approval-notification -->")) | .url' | grep -oE "[0-9]+$" | head -n 1)
|
||||
|
||||
if [ -z "$COMMENT_ID" ]; then
|
||||
gh pr comment ${{ github.event.pull_request.number }} --body "$COMMENT_BODY"
|
||||
else
|
||||
echo "Updating existing notification comment $COMMENT_ID..."
|
||||
gh api -X PATCH "repos/${{ github.repository }}/issues/comments/$COMMENT_ID" -F body="$COMMENT_BODY"
|
||||
fi
|
||||
|
||||
pr-evaluation:
|
||||
name: 'Evaluate Steering & Regressions'
|
||||
needs: 'detect-changes'
|
||||
if: "needs.detect-changes.outputs.SHOULD_RUN == 'true'"
|
||||
# Manual approval gate via environment
|
||||
environment: 'eval-gate'
|
||||
runs-on: 'gemini-cli-ubuntu-16-core'
|
||||
env:
|
||||
# CENTRALIZED MODEL LIST
|
||||
MODEL_LIST: 'gemini-3-flash-preview'
|
||||
|
||||
steps:
|
||||
- name: 'Checkout'
|
||||
uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5
|
||||
with:
|
||||
# Check out the fork's PR code for the actual evaluation
|
||||
# This only runs AFTER manual approval
|
||||
ref: '${{ github.event.pull_request.head.sha }}'
|
||||
fetch-depth: 0
|
||||
|
||||
- name: 'Remove Approval Notification'
|
||||
# Run even if other steps fail, to ensure we clean up the "Action Required" message
|
||||
if: 'always()'
|
||||
env:
|
||||
GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
|
||||
PR_NUMBER: '${{ github.event.pull_request.number }}'
|
||||
run: |
|
||||
echo "Debug: PR_NUMBER is '$PR_NUMBER'"
|
||||
# Search for the notification comment by its hidden tag
|
||||
COMMENT_ID=$(gh pr view "$PR_NUMBER" --json comments --jq '.comments[] | select(.body | contains("<!-- eval-approval-notification -->")) | .url' | grep -oE "[0-9]+$" | head -n 1)
|
||||
if [ -n "$COMMENT_ID" ]; then
|
||||
echo "Removing notification comment $COMMENT_ID now that run is approved..."
|
||||
gh api -X DELETE "repos/${{ github.repository }}/issues/comments/$COMMENT_ID"
|
||||
fi
|
||||
|
||||
- name: 'Set up Node.js'
|
||||
uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions/setup-node@v4.4.0
|
||||
with:
|
||||
node-version-file: '.nvmrc'
|
||||
cache: 'npm'
|
||||
|
||||
- name: 'Install dependencies'
|
||||
run: 'npm ci'
|
||||
|
||||
- name: 'Build project'
|
||||
run: 'npm run build'
|
||||
|
||||
- name: 'Analyze PR Content (Guidance)'
|
||||
if: "needs.detect-changes.outputs.STEERING_DETECTED == 'true'"
|
||||
id: 'analysis'
|
||||
env:
|
||||
GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
|
||||
run: |
|
||||
# Check for behavioral eval changes
|
||||
EVAL_CHANGES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD | grep "^evals/" || true)
|
||||
if [ -z "$EVAL_CHANGES" ]; then
|
||||
echo "MISSING_EVALS=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Check if user is a maintainer
|
||||
USER_PERMISSION=$(gh api repos/${{ github.repository }}/collaborators/${{ github.actor }}/permission --jq '.permission')
|
||||
if [[ "$USER_PERMISSION" == "admin" || "$USER_PERMISSION" == "write" ]]; then
|
||||
echo "IS_MAINTAINER=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: 'Execute Regression Check'
|
||||
env:
|
||||
GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
|
||||
GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
|
||||
MODEL_LIST: '${{ env.MODEL_LIST }}'
|
||||
run: |
|
||||
# Run the regression check loop. The script saves the report to a file.
|
||||
node scripts/run_eval_regression.js
|
||||
|
||||
# Use the generated report file if it exists
|
||||
if [[ -f eval_regression_report.md ]]; then
|
||||
echo "REPORT_FILE=eval_regression_report.md" >> "$GITHUB_ENV"
|
||||
fi
|
||||
|
||||
- name: 'Post or Update PR Comment'
|
||||
if: "always() && (needs.detect-changes.outputs.STEERING_DETECTED == 'true' || env.REPORT_FILE != '')"
|
||||
env:
|
||||
GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
|
||||
run: |
|
||||
# 1. Build the full comment body
|
||||
{
|
||||
if [[ -f eval_regression_report.md ]]; then
|
||||
cat eval_regression_report.md
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [[ "${{ needs.detect-changes.outputs.STEERING_DETECTED }}" == "true" ]]; then
|
||||
echo "### 🧠 Model Steering Guidance"
|
||||
echo ""
|
||||
echo "This PR modifies files that affect the model's behavior (prompts, tools, or instructions)."
|
||||
echo ""
|
||||
|
||||
if [[ "${{ steps.analysis.outputs.MISSING_EVALS }}" == "true" ]]; then
|
||||
echo "- ⚠️ **Consider adding Evals:** No behavioral evaluations (\`evals/*.eval.ts\`) were added or updated in this PR. Consider [adding a test case](https://github.com/google-gemini/gemini-cli/blob/main/evals/README.md#creating-an-evaluation) to verify the new behavior and prevent regressions."
|
||||
fi
|
||||
|
||||
if [[ "${{ steps.analysis.outputs.IS_MAINTAINER }}" == "true" ]]; then
|
||||
echo "- 🚀 **Maintainer Reminder:** Please ensure that these changes do not regress results on benchmark evals before merging."
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "---"
|
||||
echo "*This is an automated guidance message triggered by steering logic signatures.*"
|
||||
echo "<!-- eval-pr-report -->"
|
||||
} > full_comment.md
|
||||
|
||||
# 2. Find if a comment with our unique tag already exists
|
||||
# We extract the numeric ID from the URL to ensure compatibility with the REST API
|
||||
COMMENT_ID=$(gh pr view ${{ github.event.pull_request.number }} --json comments --jq '.comments[] | select(.body | contains("<!-- eval-pr-report -->")) | .url' | grep -oE "[0-9]+$" | head -n 1)
|
||||
|
||||
# 3. Update or Create the comment
|
||||
if [ -n "$COMMENT_ID" ]; then
|
||||
echo "Updating existing comment $COMMENT_ID via API..."
|
||||
gh api -X PATCH "repos/${{ github.repository }}/issues/comments/$COMMENT_ID" -F body=@full_comment.md
|
||||
else
|
||||
echo "Creating new PR comment..."
|
||||
gh pr comment ${{ github.event.pull_request.number }} --body-file full_comment.md
|
||||
fi
|
||||
@@ -0,0 +1,33 @@
|
||||
name: 'Memory Tests: Nightly'
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 2 * * *' # Runs at 2 AM every day
|
||||
workflow_dispatch: # Allow manual trigger
|
||||
|
||||
permissions:
|
||||
contents: 'read'
|
||||
|
||||
jobs:
|
||||
memory-test:
|
||||
name: 'Run Memory Usage Tests'
|
||||
runs-on: 'gemini-cli-ubuntu-16-core'
|
||||
if: "github.repository == 'google-gemini/gemini-cli'"
|
||||
steps:
|
||||
- name: 'Checkout'
|
||||
uses: 'actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8' # ratchet:actions/checkout@v5
|
||||
|
||||
- name: 'Set up Node.js'
|
||||
uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: '.nvmrc'
|
||||
cache: 'npm'
|
||||
|
||||
- name: 'Install dependencies'
|
||||
run: 'npm ci'
|
||||
|
||||
- name: 'Build project'
|
||||
run: 'npm run build'
|
||||
|
||||
- name: 'Run Memory Tests'
|
||||
run: 'npm run test:memory'
|
||||
@@ -0,0 +1,33 @@
|
||||
name: 'Performance Tests: Nightly'
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 3 * * *' # Runs at 3 AM every day
|
||||
workflow_dispatch: # Allow manual trigger
|
||||
|
||||
permissions:
|
||||
contents: 'read'
|
||||
|
||||
jobs:
|
||||
perf-test:
|
||||
name: 'Run Performance Usage Tests'
|
||||
runs-on: 'gemini-cli-ubuntu-16-core'
|
||||
if: "github.repository == 'google-gemini/gemini-cli'"
|
||||
steps:
|
||||
- name: 'Checkout'
|
||||
uses: 'actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8' # ratchet:actions/checkout@v5
|
||||
|
||||
- name: 'Set up Node.js'
|
||||
uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: '.nvmrc'
|
||||
cache: 'npm'
|
||||
|
||||
- name: 'Install dependencies'
|
||||
run: 'npm ci'
|
||||
|
||||
- name: 'Build project'
|
||||
run: 'npm run build'
|
||||
|
||||
- name: 'Run Performance Tests'
|
||||
run: 'npm run test:perf'
|
||||
Reference in New Issue
Block a user