gemini-cli/evals/frugalSearch.eval.ts

/**
 * @license
 * Copyright 2026 Google LLC
 * SPDX-License-Identifier: Apache-2.0
 */

import { describe, expect } from 'vitest';
import { evalTest } from './test-helper.js';

/**
 * Evals to verify that the agent uses search tools efficiently (frugally)
 * by utilizing limiting parameters like `total_max_matches` and `max_matches_per_file`.
 * This ensures the agent doesn't flood the context window with unnecessary search results.
 */
describe('Frugal Search', () => {
  const getGrepParams = (call: any): any => {
    let args = call.toolRequest.args;
    if (typeof args === 'string') {
      try {
        args = JSON.parse(args);
      } catch (e) {
        // Ignore parse errors
      }
    }
    return args;
  };

  evalTest('USUALLY_PASSES', {
    name: 'should use targeted search with limit',
    prompt: 'find me a sample usage of path.resolve() in the codebase',
    files: {
      'package.json': JSON.stringify({
        name: 'test-project',
        version: '1.0.0',
        main: 'dist/index.js',
        scripts: {
          build: 'tsc',
          test: 'vitest',
        },
        dependencies: {
          typescript: '^5.0.0',
          '@types/node': '^20.0.0',
          vitest: '^1.0.0',
        },
      }),
      'src/index.ts': `
        import { App } from './app.ts';
        
        const app = new App();
        app.start();
      `,
      'src/app.ts': `
        import * as path from 'path';
        import { UserController } from './controllers/user.ts';

        export class App {
          constructor() {
            console.log('App initialized');
          }

          public start(): void {
            const userController = new UserController();
            console.log('Static path:', path.resolve(__dirname, '../public'));
          }
        }
      `,
      'src/utils.ts': `
        import * as path from 'path';
        import * as fs from 'fs';

        export function resolvePath(p: string): string {
          return path.resolve(process.cwd(), p);
        }

        export function ensureDir(dirPath: string): void {
          const absolutePath = path.resolve(dirPath);
          if (!fs.existsSync(absolutePath)) {
            fs.mkdirSync(absolutePath, { recursive: true });
          }
        }
      `,
      'src/config.ts': `
        import * as path from 'path';
        
        export const config = {
          dbPath: path.resolve(process.cwd(), 'data/db.sqlite'),
          logLevel: 'info',
        };
      `,
      'src/controllers/user.ts': `
        import * as path from 'path';
        
        export class UserController {
          public getUsers(): any[] {
            console.log('Loading users from:', path.resolve('data/users.json'));
            return [{ id: 1, name: 'Alice' }];
          }
        }
      `,
      'tests/app.test.ts': `
        import { describe, it, expect } from 'vitest';
        import * as path from 'path';

        describe('App', () => {
          it('should resolve paths', () => {
            const p = path.resolve('test');
            expect(p).toBeDefined();
          });
        });
      `,
    },
    assert: async (rig) => {
      const toolCalls = rig.readToolLogs();
      const grepCalls = toolCalls.filter(
        (call) => call.toolRequest.name === 'grep_search',
      );

      expect(grepCalls.length).toBeGreaterThan(0);

      const grepParams = grepCalls.map(getGrepParams);

      const hasTotalMaxLimit = grepParams.some(
        (p) => p.total_max_matches !== undefined && p.total_max_matches <= 100,
      );
      expect(
        hasTotalMaxLimit,
        `Expected agent to use a small total_max_matches (<= 100) for a sample usage request. Actual values: ${JSON.stringify(
          grepParams.map((p) => p.total_max_matches),
        )}`,
      ).toBe(true);

      const hasMaxMatchesPerFileLimit = grepParams.some(
        (p) =>
          p.max_matches_per_file !== undefined && p.max_matches_per_file <= 5,
      );
      expect(
        hasMaxMatchesPerFileLimit,
        `Expected agent to use a small max_matches_per_file (<= 5) for a sample usage request. Actual values: ${JSON.stringify(
          grepParams.map((p) => p.max_matches_per_file),
        )}`,
      ).toBe(true);
    },
  });
});