diff --git a/.github/workflows/chained_e2e.yml b/.github/workflows/chained_e2e.yml index 2e1586bcd4..37facbda64 100644 --- a/.github/workflows/chained_e2e.yml +++ b/.github/workflows/chained_e2e.yml @@ -290,6 +290,7 @@ jobs: with: ref: '${{ needs.parse_run_context.outputs.sha }}' repository: '${{ needs.parse_run_context.outputs.repository }}' + fetch-depth: 0 - name: 'Set up Node.js 20.x' uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4 @@ -302,7 +303,14 @@ jobs: - name: 'Build project' run: 'npm run build' - - name: 'Run Evals (ALWAYS_PASSING)' + - name: 'Check if evals should run' + id: 'check_evals' + run: | + SHOULD_RUN=$(node scripts/check_evals_trigger.js) + echo "should_run=$SHOULD_RUN" >> "$GITHUB_OUTPUT" + + - name: 'Run Evals (Required to pass)' + if: steps.check_evals.outputs.should_run == 'true' env: GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}' run: 'npm run test:always_passing_evals' @@ -315,6 +323,7 @@ jobs: - 'e2e_linux' - 'e2e_mac' - 'e2e_windows' + - 'evals' - 'merge_queue_skipper' runs-on: 'gemini-cli-ubuntu-16-core' steps: @@ -322,7 +331,8 @@ jobs: run: | if [[ ${NEEDS_E2E_LINUX_RESULT} != 'success' || \ ${NEEDS_E2E_MAC_RESULT} != 'success' || \ - ${NEEDS_E2E_WINDOWS_RESULT} != 'success' ]]; then + ${NEEDS_E2E_WINDOWS_RESULT} != 'success' || \ + ${NEEDS_EVALS_RESULT} != 'success' ]]; then echo "One or more E2E jobs failed." exit 1 fi @@ -331,6 +341,7 @@ jobs: NEEDS_E2E_LINUX_RESULT: '${{ needs.e2e_linux.result }}' NEEDS_E2E_MAC_RESULT: '${{ needs.e2e_mac.result }}' NEEDS_E2E_WINDOWS_RESULT: '${{ needs.e2e_windows.result }}' + NEEDS_EVALS_RESULT: '${{ needs.evals.result }}' set_workflow_status: runs-on: 'gemini-cli-ubuntu-16-core' diff --git a/scripts/check_evals_trigger.js b/scripts/check_evals_trigger.js new file mode 100644 index 0000000000..e791b64a87 --- /dev/null +++ b/scripts/check_evals_trigger.js @@ -0,0 +1,40 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ +import { execSync } from 'node:child_process'; + +function main() { + try { + // Fetch main branch to compare against. Use || true to avoid failing if already up to date or no remote. + execSync('git fetch https://github.com/google-gemini/gemini-cli.git main', { + stdio: 'ignore', + }); + + // Find the merge base with main + const mergeBase = execSync('git merge-base HEAD FETCH_HEAD', { + encoding: 'utf-8', + }).trim(); + + // Get changed files + const changedFiles = execSync(`git diff --name-only ${mergeBase} HEAD`, { + encoding: 'utf-8', + }) + .split('\n') + .filter(Boolean); + + const shouldRun = changedFiles.some( + (file) => + file.startsWith('packages/core/src/prompts/') || + file.startsWith('packages/core/src/tools/'), + ); + + console.log(shouldRun ? 'true' : 'false'); + } catch (_error) { + // If anything fails (e.g., no git history), run evals to be safe + console.log('true'); + } +} + +main();