mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-04-29 22:44:45 -07:00
fix(workflows): tune triage prompt and add robustness evals
This commit is contained in:
cocosheng-g
@@ -0,0 +1,204 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, expect } from 'vitest';
|
||||
import { evalTest } from './test-helper.js';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
// Read the workflow file to extract the prompt
|
||||
const workflowPath = path.join(
|
||||
process.cwd(),
|
||||
'.github/workflows/gemini-automated-issue-triage.yml',
|
||||
);
|
||||
const workflowContent = fs.readFileSync(workflowPath, 'utf8');
|
||||
|
||||
// Extract the prompt block
|
||||
// Looking for "prompt: |-" followed by the content, until the next step definition
|
||||
const promptMatch = workflowContent.match(
|
||||
/prompt: \|-\n([\s\S]+?)(?=\n\s+-\s+name:)/,
|
||||
);
|
||||
|
||||
if (!promptMatch) {
|
||||
throw new Error(
|
||||
'Could not extract prompt from workflow file. Check regex or file content.',
|
||||
);
|
||||
}
|
||||
|
||||
const rawPrompt = promptMatch[1];
|
||||
// Remove the YAML indentation (12 spaces based on the file structure)
|
||||
// We detect the indentation from the first line
|
||||
const lines = rawPrompt.split('\n');
|
||||
const firstLineIndent = lines[0].match(/^\s*/)?.[0].length || 0;
|
||||
const TRIAGE_PROMPT_TEMPLATE = lines
|
||||
.map((line) => line.slice(firstLineIndent))
|
||||
.join('\n');
|
||||
|
||||
const createPrompt = (title: string, body: string) => {
|
||||
// The placeholders in the YAML are ${{ env.ISSUE_TITLE }} etc.
|
||||
// We need to replace them with the actual values for the test.
|
||||
return TRIAGE_PROMPT_TEMPLATE.replace('${{ env.ISSUE_TITLE }}', title)
|
||||
.replace('${{ env.ISSUE_BODY }}', body)
|
||||
.replace(
|
||||
'${{ env.AVAILABLE_LABELS }}',
|
||||
'area/agent, area/enterprise, area/non-interactive, area/core, area/security, area/platform, area/extensions, area/unknown',
|
||||
);
|
||||
};
|
||||
|
||||
describe('triage_agent', () => {
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'should identify area/core for windows installation issues',
|
||||
prompt: createPrompt(
|
||||
'CLI failed to install on Windows',
|
||||
'I tried running npm install but it failed with an error on Windows 11.',
|
||||
),
|
||||
assert: async (rig, result) => {
|
||||
const json = JSON.parse(result.match(/{[\s\S]*}/)?.[0] || '{}');
|
||||
expect(json.labels_to_set).toContain('area/core');
|
||||
},
|
||||
});
|
||||
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'should identify area/platform for CI/CD failures',
|
||||
prompt: createPrompt(
|
||||
'Tests are failing in the CI/CD pipeline',
|
||||
'The github action is failing with a 500 error.',
|
||||
),
|
||||
assert: async (rig, result) => {
|
||||
const json = JSON.parse(result.match(/{[\s\S]*}/)?.[0] || '{}');
|
||||
expect(json.labels_to_set).toContain('area/platform');
|
||||
},
|
||||
});
|
||||
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'should identify area/platform for quota issues',
|
||||
prompt: createPrompt(
|
||||
'Resource Exhausted 429',
|
||||
'I am getting a 429 error when running the CLI.',
|
||||
),
|
||||
assert: async (rig, result) => {
|
||||
const json = JSON.parse(result.match(/{[\s\S]*}/)?.[0] || '{}');
|
||||
expect(json.labels_to_set).toContain('area/platform');
|
||||
},
|
||||
});
|
||||
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'should identify area/core for local build failures',
|
||||
prompt: createPrompt(
|
||||
'Local build failing',
|
||||
'I cannot build the project locally. npm run build fails.',
|
||||
),
|
||||
assert: async (rig, result) => {
|
||||
const json = JSON.parse(result.match(/{[\s\S]*}/)?.[0] || '{}');
|
||||
expect(json.labels_to_set).toContain('area/core');
|
||||
},
|
||||
});
|
||||
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'should identify area/platform for sandbox issues',
|
||||
prompt: createPrompt(
|
||||
'Sandbox connection failed',
|
||||
'I cannot connect to the docker sandbox environment.',
|
||||
),
|
||||
assert: async (rig, result) => {
|
||||
const json = JSON.parse(result.match(/{[\s\S]*}/)?.[0] || '{}');
|
||||
expect(json.labels_to_set).toContain('area/platform');
|
||||
},
|
||||
});
|
||||
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'should identify area/core for local test failures',
|
||||
prompt: createPrompt(
|
||||
'Local tests failing',
|
||||
'I am running npm test locally and it fails.',
|
||||
),
|
||||
assert: async (rig, result) => {
|
||||
const json = JSON.parse(result.match(/{[\s\S]*}/)?.[0] || '{}');
|
||||
expect(json.labels_to_set).toContain('area/core');
|
||||
},
|
||||
});
|
||||
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'should identify area/agent for questions about tools',
|
||||
prompt: createPrompt(
|
||||
'Bug with web search?',
|
||||
'I am trying to use web search but I do not know the syntax. Is it @web or /web?',
|
||||
),
|
||||
assert: async (rig, result) => {
|
||||
const json = JSON.parse(result.match(/{[\s\S]*}/)?.[0] || '{}');
|
||||
expect(json.labels_to_set).toContain('area/agent');
|
||||
},
|
||||
});
|
||||
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'should identify area/extensions for feature requests',
|
||||
prompt: createPrompt(
|
||||
'Please add a python extension',
|
||||
'I want to write python scripts as an extension.',
|
||||
),
|
||||
assert: async (rig, result) => {
|
||||
const json = JSON.parse(result.match(/{[\s\S]*}/)?.[0] || '{}');
|
||||
expect(json.labels_to_set).toContain('area/extensions');
|
||||
},
|
||||
});
|
||||
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'should identify area/unknown for off-topic spam',
|
||||
prompt: createPrompt('Buy cheap rolex', 'Click here for discount.'),
|
||||
assert: async (rig, result) => {
|
||||
const json = JSON.parse(result.match(/{[\s\S]*}/)?.[0] || '{}');
|
||||
expect(json.labels_to_set).toContain('area/unknown');
|
||||
},
|
||||
});
|
||||
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'should identify area/core for crash reports phrased as questions',
|
||||
prompt: createPrompt(
|
||||
'Why does it segfault?',
|
||||
'Why does the CLI segfault immediately when I run it on Ubuntu?',
|
||||
),
|
||||
assert: async (rig, result) => {
|
||||
const json = JSON.parse(result.match(/{[\s\S]*}/)?.[0] || '{}');
|
||||
expect(json.labels_to_set).toContain('area/core');
|
||||
},
|
||||
});
|
||||
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'should identify area/agent for feature requests for built-in tools',
|
||||
prompt: createPrompt(
|
||||
'Can we have a diff tool?',
|
||||
'Is it possible to add a built-in tool to show diffs before editing?',
|
||||
),
|
||||
assert: async (rig, result) => {
|
||||
const json = JSON.parse(result.match(/{[\s\S]*}/)?.[0] || '{}');
|
||||
expect(json.labels_to_set).toContain('area/agent');
|
||||
},
|
||||
});
|
||||
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'should identify area/enterprise for license questions',
|
||||
prompt: createPrompt(
|
||||
'License key issue',
|
||||
'Where do I enter my enterprise license key? I cannot find the setting.',
|
||||
),
|
||||
assert: async (rig, result) => {
|
||||
const json = JSON.parse(result.match(/{[\s\S]*}/)?.[0] || '{}');
|
||||
expect(json.labels_to_set).toContain('area/enterprise');
|
||||
},
|
||||
});
|
||||
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'should identify area/unknown for extremely vague reports',
|
||||
prompt: createPrompt(
|
||||
'It does not work',
|
||||
'I tried to use it and it failed.',
|
||||
),
|
||||
assert: async (rig, result) => {
|
||||
const json = JSON.parse(result.match(/{[\s\S]*}/)?.[0] || '{}');
|
||||
expect(json.labels_to_set).toContain('area/unknown');
|
||||
},
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user