MediaMetz/gemini-cli
1
0
Fork 0
mirror of https://github.com/google-gemini/gemini-cli.git synced 2026-05-12 21:03:05 -07:00
Code Issues Packages Projects Releases Wiki Activity

fix(test): move flaky tests to non-blocking suite (#23259)

Browse Source
This commit is contained in:
matt korwel
2026-03-23 13:54:47 -07:00
committed by GitHub
parent 15f8026983
commit b10bcf49b9
4 changed files with 2272 additions and 2239 deletions
Download Patch File Download Diff File Expand all files Collapse all files
integration-tests/hooks-system.test.ts
+45 -18
View File
@@ -5,12 +5,15 @@
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { TestRig, poll, normalizePath } from './test-helper.js';
import { TestRig, poll, normalizePath, skipFlaky } from './test-helper.js';
import { join } from 'node:path';
import { writeFileSync, existsSync, mkdirSync } from 'node:fs';
import os from 'node:os';
describe('Hooks System Integration', { timeout: 120000 }, () => {
describe.skipIf(skipFlaky)(
'Hooks System Integration',
{ timeout: 120000 },
() => {
let rig: TestRig;
beforeEach(() => {
@@ -74,7 +77,8 @@ describe('Hooks System Integration', { timeout: 120000 }, () => {
const toolLogs = rig.readToolLogs();
const writeFileCalls = toolLogs.filter(
(t) =>
t.toolRequest.name === 'write_file' && t.toolRequest.success === true,
t.toolRequest.name === 'write_file' &&
t.toolRequest.success === true,
);
// Tool should not be called due to blocking hook
@@ -140,7 +144,8 @@ describe('Hooks System Integration', { timeout: 120000 }, () => {
const toolLogs = rig.readToolLogs();
const writeFileCalls = toolLogs.filter(
(t) =>
t.toolRequest.name === 'write_file' && t.toolRequest.success === true,
t.toolRequest.name === 'write_file' &&
t.toolRequest.success === true,
);
// Tool should not be called due to blocking hook
@@ -158,9 +163,9 @@ describe('Hooks System Integration', { timeout: 120000 }, () => {
log.hookCall.stderr.includes('"decision":"deny"')),
);
expect(blockHook).toBeDefined();
expect(blockHook?.hookCall.stdout + blockHook?.hookCall.stderr).toContain(
blockMsg,
);
expect(
blockHook?.hookCall.stdout + blockHook?.hookCall.stderr,
).toContain(blockMsg);
});
it('should allow tool execution when hook returns allow decision', async () => {
@@ -275,7 +280,9 @@ describe('Hooks System Integration', { timeout: 120000 }, () => {
// Should generate hook telemetry
const hookTelemetryFound = rig.readHookLogs();
expect(hookTelemetryFound.length).toBeGreaterThan(0);
expect(hookTelemetryFound[0].hookCall.hook_event_name).toBe('AfterTool');
expect(hookTelemetryFound[0].hookCall.hook_event_name).toBe(
'AfterTool',
);
expect(hookTelemetryFound[0].hookCall.hook_name).toBe(
normalizePath(command),
);
@@ -420,7 +427,10 @@ console.log(JSON.stringify({
}
}));`;
const scriptPath = rig.createScript('before_model_hook.cjs', hookScript);
const scriptPath = rig.createScript(
'before_model_hook.cjs',
hookScript,
);
rig.setup('should modify LLM requests with BeforeModel hooks', {
settings: {
@@ -599,7 +609,10 @@ console.log(JSON.stringify({
}
}));`;
const scriptPath = rig.createScript('after_model_hook.cjs', hookScript);
const scriptPath = rig.createScript(
'after_model_hook.cjs',
hookScript,
);
rig.setup('should modify LLM responses with AfterModel hooks', {
settings: {
@@ -630,7 +643,8 @@ console.log(JSON.stringify({
);
// Should generate hook telemetry
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
const hookTelemetryFound =
await rig.waitForTelemetryEvent('hook_call');
expect(hookTelemetryFound).toBeTruthy();
},
);
@@ -725,7 +739,10 @@ console.log(JSON.stringify({
),
});
const scriptPath = rig.createScript('before_agent_hook.cjs', hookScript);
const scriptPath = rig.createScript(
'before_agent_hook.cjs',
hookScript,
);
rig.setup('should augment prompts with BeforeAgent hooks', {
settings: {
@@ -821,7 +838,8 @@ console.log(JSON.stringify({
await run.expectText('test', 10000);
// Should find the shell command execution
const foundShellCommand = await rig.waitForToolCall('run_shell_command');
const foundShellCommand =
await rig.waitForToolCall('run_shell_command');
expect(foundShellCommand).toBeTruthy();
// Verify Notification hook executed
@@ -1663,7 +1681,9 @@ console.log(JSON.stringify({
// If polling failed, log diagnostic info
if (!pollResult) {
const hookLogs = rig.readHookLogs();
const hookEvents = hookLogs.map((log) => log.hookCall.hook_event_name);
const hookEvents = hookLogs.map(
(log) => log.hookCall.hook_event_name,
);
console.error(
`Polling timeout after 90000ms: Expected >= ${expectedMinHooks} hooks, got ${hookLogs.length}`,
);
@@ -2111,7 +2131,10 @@ console.log(JSON.stringify({
hookOutput,
)}));`;
const scriptPath = rig.createScript('input_override_hook.js', hookScript);
const scriptPath = rig.createScript(
'input_override_hook.js',
hookScript,
);
// 2. Full setup with settings and fake responses
rig.setup('should override tool input parameters via BeforeTool hook', {
@@ -2242,7 +2265,8 @@ console.log(JSON.stringify({
const toolLogs = rig.readToolLogs();
const writeFileCalls = toolLogs.filter(
(t) =>
t.toolRequest.name === 'write_file' && t.toolRequest.success === true,
t.toolRequest.name === 'write_file' &&
t.toolRequest.success === true,
);
expect(writeFileCalls).toHaveLength(0);
});
@@ -2326,7 +2350,9 @@ console.log(JSON.stringify({
await run.expectText('Type your message', 30000);
// Send prompt that will trigger write_file
await run.type('Create a file called ask-test.txt with content "test"');
await run.type(
'Create a file called ask-test.txt with content "test"',
);
await run.type('\r');
// Wait for the FORCED confirmation prompt to appear
@@ -2453,4 +2479,5 @@ console.log(JSON.stringify({
},
);
});
});
},
);
integration-tests/symlink-install.test.ts
+6 -3
View File
@@ -5,7 +5,7 @@
*/
import { describe, expect, it, beforeEach, afterEach } from 'vitest';
import { TestRig, InteractiveRun } from './test-helper.js';
import { TestRig, InteractiveRun, skipFlaky } from './test-helper.js';
import * as fs from 'node:fs';
import * as os from 'node:os';
import {
@@ -33,7 +33,9 @@ const otherExtension = `{
"version": "6.6.6"
}`;
describe('extension symlink install spoofing protection', () => {
describe.skipIf(skipFlaky)(
'extension symlink install spoofing protection',
() => {
let rig: TestRig;
beforeEach(() => {
@@ -133,4 +135,5 @@ describe('extension symlink install spoofing protection', () => {
await run2.expectText('Installation aborted', 30000);
await run2.kill();
}, 60000);
});
},
);
integration-tests/test-helper.ts
+2
View File
@@ -6,3 +6,5 @@
export * from '@google/gemini-cli-test-utils';
export { normalizePath } from '@google/gemini-cli-test-utils';
export const skipFlaky = !process.env['RUN_FLAKY_INTEGRATION'];
package.json
+1
View File
@@ -48,6 +48,7 @@
"test:all_evals": "cross-env RUN_EVALS=1 vitest run --config evals/vitest.config.ts",
"test:e2e": "cross-env VERBOSE=true KEEP_OUTPUT=true npm run test:integration:sandbox:none",
"test:integration:all": "npm run test:integration:sandbox:none && npm run test:integration:sandbox:docker && npm run test:integration:sandbox:podman",
"test:integration:flaky": "cross-env RUN_FLAKY_INTEGRATION=1 npm run test:integration:sandbox:none",
"test:integration:sandbox:none": "cross-env GEMINI_SANDBOX=false vitest run --root ./integration-tests",
"test:integration:sandbox:docker": "cross-env GEMINI_SANDBOX=docker npm run build:sandbox && cross-env GEMINI_SANDBOX=docker vitest run --root ./integration-tests",
"test:integration:sandbox:podman": "cross-env GEMINI_SANDBOX=podman vitest run --root ./integration-tests",