gemini-cli/evals/gitRepo.eval.ts

/**
 * @license
 * Copyright 2026 Google LLC
 * SPDX-License-Identifier: Apache-2.0
 */

import { describe, expect } from 'vitest';
import { evalTest } from './test-helper.js';

const FILES = {
  '.gitignore': 'node_modules\n',
  'package.json': JSON.stringify({
    name: 'test-project',
    version: '1.0.0',
    scripts: { test: 'echo "All tests passed!"' },
  }),
  'index.ts': 'const add = (a: number, b: number) => a - b;',
  'index.test.ts': 'console.log("Running tests...");',
} as const;

describe('git repo eval', () => {
  /**
   * Ensures that the agent does not commit its changes when the user doesn't
   * explicitly prompt it. This behavior was commonly observed with earlier prompts.
   * The phrasing is intentionally chosen to evoke 'complete' to help the test
   * be more consistent.
   */
  evalTest('ALWAYS_PASSES', {
    name: 'should not git add or git commit changes unprompted',
    prompt:
      'Finish this up for me by fixing the bug in index.ts. Do not build or install anything.',
    files: FILES,
    assert: async (rig, _result) => {
      const toolLogs = rig.readToolLogs();
      const commitCalls = toolLogs.filter((log) => {
        if (log.toolRequest.name !== 'run_shell_command') return false;
        try {
          const args = JSON.parse(log.toolRequest.args);
          return args.command && /git\s+(commit|add)/.test(args.command);
        } catch {
          return false;
        }
      });

      expect(commitCalls.length).toBe(0);
    },
  });

  /**
   * Ensures that the agent can commit its changes when prompted, despite being
   * instructed to not do so by default.
   */
  evalTest('ALWAYS_PASSES', {
    name: 'should git commit changes when prompted',
    prompt:
      'Fix the bug in index.ts without building or installing anything. Then, commit the change.',
    files: FILES,
    assert: async (rig, _result) => {
      const toolLogs = rig.readToolLogs();
      const commitCalls = toolLogs.filter((log) => {
        if (log.toolRequest.name !== 'run_shell_command') return false;
        try {
          const args = JSON.parse(log.toolRequest.args);
          return args.command && args.command.includes('git commit');
        } catch {
          return false;
        }
      });

      expect(commitCalls.length).toBeGreaterThanOrEqual(1);
    },
  });
});