From 035bea3699f127f5c579c49df60843c9a28452a4 Mon Sep 17 00:00:00 2001 From: Tommaso Sciortino Date: Tue, 2 Dec 2025 17:43:06 -0800 Subject: [PATCH] Restrict integration tests tools (#14403) --- integration-tests/file-system.test.ts | 17 ++++++++++--- integration-tests/google_web_search.test.ts | 4 ++- integration-tests/list_directory.test.ts | 4 ++- integration-tests/read_many_files.test.ts | 4 ++- integration-tests/replace.test.ts | 13 +++++++--- integration-tests/run_shell_command.test.ts | 27 ++++++++++++++++----- integration-tests/save_memory.test.ts | 4 ++- integration-tests/simple-mcp-server.test.ts | 1 + integration-tests/utf-bom-encoding.test.ts | 4 ++- integration-tests/write_file.test.ts | 4 ++- 10 files changed, 63 insertions(+), 19 deletions(-) diff --git a/integration-tests/file-system.test.ts b/integration-tests/file-system.test.ts index a056a2d1dd..cbdd5456b6 100644 --- a/integration-tests/file-system.test.ts +++ b/integration-tests/file-system.test.ts @@ -12,7 +12,9 @@ import { TestRig, printDebugInfo, validateModelOutput } from './test-helper.js'; describe('file-system', () => { it('should be able to read a file', async () => { const rig = new TestRig(); - await rig.setup('should be able to read a file'); + await rig.setup('should be able to read a file', { + settings: { tools: { core: ['read_file'] } }, + }); rig.createFile('test.txt', 'hello world'); const result = await rig.run( @@ -40,7 +42,9 @@ describe('file-system', () => { it('should be able to write a file', async () => { const rig = new TestRig(); - await rig.setup('should be able to write a file'); + await rig.setup('should be able to write a file', { + settings: { tools: { core: ['write_file', 'replace', 'read_file'] } }, + }); rig.createFile('test.txt', ''); const result = await rig.run(`edit test.txt to have a hello world message`); @@ -95,7 +99,9 @@ describe('file-system', () => { it('should correctly handle file paths with spaces', async () => { const rig = new TestRig(); - await rig.setup('should correctly handle file paths with spaces'); + await rig.setup('should correctly handle file paths with spaces', { + settings: { tools: { core: ['write_file', 'read_file'] } }, + }); const fileName = 'my test file.txt'; const result = await rig.run( @@ -117,7 +123,9 @@ describe('file-system', () => { it('should perform a read-then-write sequence', async () => { const rig = new TestRig(); - await rig.setup('should perform a read-then-write sequence'); + await rig.setup('should perform a read-then-write sequence', { + settings: { tools: { core: ['read_file', 'replace', 'write_file'] } }, + }); const fileName = 'version.txt'; rig.createFile(fileName, '1.0.0'); @@ -206,6 +214,7 @@ describe('file-system', () => { const rig = new TestRig(); await rig.setup( 'should fail safely when trying to edit a non-existent file', + { settings: { tools: { core: ['read_file', 'replace'] } } }, ); const fileName = 'non_existent.txt'; diff --git a/integration-tests/google_web_search.test.ts b/integration-tests/google_web_search.test.ts index c3c9003932..9c0fc41d94 100644 --- a/integration-tests/google_web_search.test.ts +++ b/integration-tests/google_web_search.test.ts @@ -11,7 +11,9 @@ import { TestRig, printDebugInfo, validateModelOutput } from './test-helper.js'; describe(WEB_SEARCH_TOOL_NAME, () => { it('should be able to search the web', async () => { const rig = new TestRig(); - await rig.setup('should be able to search the web'); + await rig.setup('should be able to search the web', { + settings: { tools: { core: [WEB_SEARCH_TOOL_NAME] } }, + }); let result; try { diff --git a/integration-tests/list_directory.test.ts b/integration-tests/list_directory.test.ts index 1da05b9db6..4b3f2d59b6 100644 --- a/integration-tests/list_directory.test.ts +++ b/integration-tests/list_directory.test.ts @@ -17,7 +17,9 @@ import { join } from 'node:path'; describe('list_directory', () => { it('should be able to list a directory', async () => { const rig = new TestRig(); - rig.setup('should be able to list a directory'); + await rig.setup('should be able to list a directory', { + settings: { tools: { core: ['list_directory'] } }, + }); rig.createFile('file1.txt', 'file 1 content'); rig.mkdir('subdir'); rig.sync(); diff --git a/integration-tests/read_many_files.test.ts b/integration-tests/read_many_files.test.ts index 1d80092ce6..bc5b26c599 100644 --- a/integration-tests/read_many_files.test.ts +++ b/integration-tests/read_many_files.test.ts @@ -10,7 +10,9 @@ import { TestRig, printDebugInfo, validateModelOutput } from './test-helper.js'; describe('read_many_files', () => { it.skip('should be able to read multiple files', async () => { const rig = new TestRig(); - await rig.setup('should be able to read multiple files'); + await rig.setup('should be able to read multiple files', { + settings: { tools: { core: ['read_many_files', 'read_file'] } }, + }); rig.createFile('file1.txt', 'file 1 content'); rig.createFile('file2.txt', 'file 2 content'); diff --git a/integration-tests/replace.test.ts b/integration-tests/replace.test.ts index 4b0eaeddff..ab6f39c7b9 100644 --- a/integration-tests/replace.test.ts +++ b/integration-tests/replace.test.ts @@ -10,7 +10,9 @@ import { TestRig } from './test-helper.js'; describe('replace', () => { it('should be able to replace content in a file', async () => { const rig = new TestRig(); - await rig.setup('should be able to replace content in a file'); + await rig.setup('should be able to replace content in a file', { + settings: { tools: { core: ['replace', 'read_file'] } }, + }); const fileName = 'file_to_replace.txt'; const originalContent = 'foo content'; @@ -30,6 +32,7 @@ describe('replace', () => { const rig = new TestRig(); await rig.setup( 'should handle $ literally when replacing text ending with $', + { settings: { tools: { core: ['replace', 'read_file'] } } }, ); const fileName = 'regex.yml'; @@ -50,7 +53,9 @@ describe('replace', () => { it.skip('should insert a multi-line block of text', async () => { const rig = new TestRig(); - await rig.setup('should insert a multi-line block of text'); + await rig.setup('should insert a multi-line block of text', { + settings: { tools: { core: ['replace', 'read_file'] } }, + }); const fileName = 'insert_block.txt'; const originalContent = 'Line A\n\nLine C'; const newBlock = 'First line\nSecond line\nThird line'; @@ -69,7 +74,9 @@ describe('replace', () => { it.skip('should delete a block of text', async () => { const rig = new TestRig(); - await rig.setup('should delete a block of text'); + await rig.setup('should delete a block of text', { + settings: { tools: { core: ['replace', 'read_file'] } }, + }); const fileName = 'delete_block.txt'; const blockToDelete = '## DELETE THIS ##\nThis is a block of text to delete.\n## END DELETE ##'; diff --git a/integration-tests/run_shell_command.test.ts b/integration-tests/run_shell_command.test.ts index edd1d89559..855c744a83 100644 --- a/integration-tests/run_shell_command.test.ts +++ b/integration-tests/run_shell_command.test.ts @@ -86,7 +86,9 @@ function getChainedEchoCommand(): { allowPattern: string; command: string } { describe('run_shell_command', () => { it('should be able to run a shell command', async () => { const rig = new TestRig(); - await rig.setup('should be able to run a shell command'); + await rig.setup('should be able to run a shell command', { + settings: { tools: { core: ['run_shell_command'] } }, + }); const prompt = `Please run the command "echo hello-world" and show me the output`; @@ -118,7 +120,9 @@ describe('run_shell_command', () => { it('should be able to run a shell command via stdin', async () => { const rig = new TestRig(); - await rig.setup('should be able to run a shell command via stdin'); + await rig.setup('should be able to run a shell command via stdin', { + settings: { tools: { core: ['run_shell_command'] } }, + }); const prompt = `Please run the command "echo test-stdin" and show me what it outputs`; @@ -230,7 +234,9 @@ describe('run_shell_command', () => { it('should succeed with --yolo mode', async () => { const rig = new TestRig(); - await rig.setup('should succeed with --yolo mode'); + await rig.setup('should succeed with --yolo mode', { + settings: { tools: { core: ['run_shell_command'] } }, + }); const testFile = rig.createFile('test.txt', 'Lorem\nIpsum\nDolor\n'); const { command } = getLineCountCommand(); @@ -362,7 +368,9 @@ describe('run_shell_command', () => { it('should reject commands not on the allowlist', async () => { const rig = new TestRig(); - await rig.setup('should reject commands not on the allowlist'); + await rig.setup('should reject commands not on the allowlist', { + settings: { tools: { core: ['run_shell_command'] } }, + }); const testFile = rig.createFile('test.txt', 'Disallowed command check\n'); const allowedCommand = getAllowedListCommand(); @@ -461,6 +469,9 @@ describe('run_shell_command', () => { const rig = new TestRig(); await rig.setup( 'should allow all with "ShellTool" and other specific tools', + { + settings: { tools: { core: ['run_shell_command'] } }, + }, ); const { tool } = getLineCountCommand(); @@ -506,7 +517,9 @@ describe('run_shell_command', () => { it('should propagate environment variables to the child process', async () => { const rig = new TestRig(); - await rig.setup('should propagate environment variables'); + await rig.setup('should propagate environment variables', { + settings: { tools: { core: ['run_shell_command'] } }, + }); const varName = 'GEMINI_CLI_TEST_VAR'; const varValue = `test-value-${Math.random().toString(36).substring(7)}`; @@ -566,7 +579,9 @@ describe('run_shell_command', () => { it('rejects invalid shell expressions', async () => { const rig = new TestRig(); - await rig.setup('rejects invalid shell expressions'); + await rig.setup('rejects invalid shell expressions', { + settings: { tools: { core: ['run_shell_command'] } }, + }); const invalidCommand = getInvalidCommand(); const result = await rig.run( `I am testing the error handling of the run_shell_command tool. Please attempt to run the following command, which I know has invalid syntax: \`${invalidCommand}\`. If the command fails as expected, please return the word FAIL, otherwise return the word SUCCESS.`, diff --git a/integration-tests/save_memory.test.ts b/integration-tests/save_memory.test.ts index 15b062e9a2..f841fce274 100644 --- a/integration-tests/save_memory.test.ts +++ b/integration-tests/save_memory.test.ts @@ -10,7 +10,9 @@ import { TestRig, printDebugInfo, validateModelOutput } from './test-helper.js'; describe('save_memory', () => { it('should be able to save to memory', async () => { const rig = new TestRig(); - await rig.setup('should be able to save to memory'); + await rig.setup('should be able to save to memory', { + settings: { tools: { core: ['save_memory'] } }, + }); const prompt = `remember that my favorite color is blue. diff --git a/integration-tests/simple-mcp-server.test.ts b/integration-tests/simple-mcp-server.test.ts index 0546514467..d94af46648 100644 --- a/integration-tests/simple-mcp-server.test.ts +++ b/integration-tests/simple-mcp-server.test.ts @@ -177,6 +177,7 @@ describe('simple-mcp-server', () => { args: ['mcp-server.cjs'], }, }, + tools: { core: [] }, }, }); diff --git a/integration-tests/utf-bom-encoding.test.ts b/integration-tests/utf-bom-encoding.test.ts index a870045453..c382a06fb1 100644 --- a/integration-tests/utf-bom-encoding.test.ts +++ b/integration-tests/utf-bom-encoding.test.ts @@ -53,7 +53,9 @@ let dir: string; describe('BOM end-to-end integraion', () => { beforeAll(async () => { rig = new TestRig(); - await rig.setup('bom-integration'); + await rig.setup('bom-integration', { + settings: { tools: { core: ['read_file'] } }, + }); dir = rig.testDir!; }); diff --git a/integration-tests/write_file.test.ts b/integration-tests/write_file.test.ts index a53ef9407e..c0ecd1b8e2 100644 --- a/integration-tests/write_file.test.ts +++ b/integration-tests/write_file.test.ts @@ -15,7 +15,9 @@ import { describe('write_file', () => { it('should be able to write a file', async () => { const rig = new TestRig(); - await rig.setup('should be able to write a file'); + await rig.setup('should be able to write a file', { + settings: { tools: { core: ['write_file', 'read_file'] } }, + }); const prompt = `show me an example of using the write tool. put a dad joke in dad.txt`; const result = await rig.run(prompt);