feat(skills): add experimentation skill and metadata

Adds a new experimentation skill to allow users to easily manage, view, and override remote Gemini CLI experiments locally. This also introduces `ExperimentMetadata` to `flagNames.ts` to expose descriptions and default values for each experiment flag.
This commit is contained in:
mkorwel
2026-02-19 14:58:03 -06:00
committed by Matt Korwel
parent 0179726222
commit 50f71a8df5
4 changed files with 182 additions and 0 deletions
+46
View File
@@ -0,0 +1,46 @@
---
name: experimentation
description: Manage, view, and override Gemini CLI remote experiments and feature flags locally.
---
# Experimentation Skill
This skill allows you to safely manage, view, and locally override remote Gemini CLI experiments (feature flags).
## Core Concepts
Remote experimentation is enabled by default. `packages/core/src/code_assist/experiments/flagNames.ts` contains the active `ExperimentFlags` and `ExperimentMetadata` which describe each flag's purpose, type, and default value.
Currently, Gemini CLI supports local overrides using the `GEMINI_EXP` environment variable pointing to a JSON file.
## Workflow: Viewing Experiments
When a user asks what experiments are active or available:
1. Search `packages/core/src/code_assist/experiments/flagNames.ts` for `ExperimentMetadata`.
2. Extract the descriptions, types, and defaults to answer the user's questions.
3. Check if there is an active local override file at `.gemini/experiments.json`.
## Workflow: Local Overrides & Opt-Out
When a user wants to override a flag locally (e.g., to turn off a preview feature) or opt-out:
1. Use the `scripts/override_experiment.cjs` script bundled with this skill to safely update or create `.gemini/experiments.json`.
2. When the user asks to "opt out of experiments", use the script to set `experimentIds` to an empty array and clear flags, ensuring a sensible baseline.
3. **Important:** After updating the JSON file, instruct the user to run the CLI with `GEMINI_EXP` set, e.g.:
`GEMINI_EXP=.gemini/experiments.json gemini <command>`
## Using the Scripts
You have access to `scripts/override_experiment.cjs` to manage the local overrides safely without disrupting the file structure required by the CLI backend.
Example usage:
```bash
# Enable an experiment locally
node .gemini/skills/experimentation/scripts/override_experiment.cjs set 45740196 true
# Remove an override
node .gemini/skills/experimentation/scripts/override_experiment.cjs unset 45740196
# Opt out of all experiments (clear everything)
node .gemini/skills/experimentation/scripts/override_experiment.cjs clear
```
@@ -0,0 +1,72 @@
const fs = require('fs');
const path = require('path');
const filePath = path.join(process.cwd(), '.gemini', 'experiments.json');
function readExperiments() {
if (fs.existsSync(filePath)) {
try {
const data = JSON.parse(fs.readFileSync(filePath, 'utf8'));
return {
flags: data.flags || [],
experimentIds: data.experimentIds || []
};
} catch (e) {
console.error('Failed to parse existing experiments.json, starting fresh.');
}
}
return { flags: [], experimentIds: [] };
}
function writeExperiments(data) {
const dir = path.dirname(filePath);
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf8');
console.log(`Updated local experiments at ${filePath}`);
}
const args = process.argv.slice(2);
const command = args[0];
if (!command) {
console.error('Usage: override_experiment.cjs <set|unset|clear> [flagId] [value]');
process.exit(1);
}
const data = readExperiments();
if (command === 'clear') {
data.flags = [];
data.experimentIds = [];
writeExperiments(data);
} else if (command === 'unset') {
const flagId = parseInt(args[1], 10);
data.flags = data.flags.filter(f => f.flagId !== flagId);
writeExperiments(data);
} else if (command === 'set') {
const flagId = parseInt(args[1], 10);
const rawValue = args[2];
if (isNaN(flagId) || rawValue === undefined) {
console.error('Invalid arguments for set: requires numeric flagId and value');
process.exit(1);
}
// Remove existing flag
data.flags = data.flags.filter(f => f.flagId !== flagId);
// Parse value
const flag = { flagId };
if (rawValue === 'true') flag.boolValue = true;
else if (rawValue === 'false') flag.boolValue = false;
else if (!isNaN(Number(rawValue))) flag.numberValue = Number(rawValue);
else flag.stringValue = rawValue;
data.flags.push(flag);
writeExperiments(data);
} else {
console.error('Unknown command');
process.exit(1);
}
Binary file not shown.
@@ -24,3 +24,67 @@ export const ExperimentFlags = {
export type ExperimentFlagName =
(typeof ExperimentFlags)[keyof typeof ExperimentFlags];
export interface ExperimentMetadataEntry {
description: string;
type: 'boolean' | 'number' | 'string';
defaultValue: boolean | number | string;
}
export const ExperimentMetadata: Record<number, ExperimentMetadataEntry> = {
[ExperimentFlags.CONTEXT_COMPRESSION_THRESHOLD]: {
description: 'Threshold at which context compression activates.',
type: 'number',
defaultValue: 0,
},
[ExperimentFlags.USER_CACHING]: {
description: 'Enables caching of user contexts.',
type: 'boolean',
defaultValue: false,
},
[ExperimentFlags.BANNER_TEXT_NO_CAPACITY_ISSUES]: {
description: 'Banner text displayed when there are no capacity issues.',
type: 'string',
defaultValue: '',
},
[ExperimentFlags.BANNER_TEXT_CAPACITY_ISSUES]: {
description: 'Banner text displayed during capacity issues.',
type: 'string',
defaultValue: '',
},
[ExperimentFlags.ENABLE_PREVIEW]: {
description: 'Enables preview features globally.',
type: 'boolean',
defaultValue: false,
},
[ExperimentFlags.ENABLE_NUMERICAL_ROUTING]: {
description: 'Enables numerical routing strategies for the model.',
type: 'boolean',
defaultValue: false,
},
[ExperimentFlags.CLASSIFIER_THRESHOLD]: {
description: 'Threshold for the intent classifier.',
type: 'number',
defaultValue: 0.5,
},
[ExperimentFlags.ENABLE_ADMIN_CONTROLS]: {
description: 'Enables admin control features in the CLI.',
type: 'boolean',
defaultValue: false,
},
[ExperimentFlags.MASKING_PROTECTION_THRESHOLD]: {
description: 'Threshold for masking protection logic.',
type: 'number',
defaultValue: 0,
},
[ExperimentFlags.MASKING_PRUNABLE_THRESHOLD]: {
description: 'Threshold for prunable masking.',
type: 'number',
defaultValue: 0,
},
[ExperimentFlags.MASKING_PROTECT_LATEST_TURN]: {
description: 'Protects the latest turn from being masked.',
type: 'boolean',
defaultValue: true,
},
};