MediaMetz/gemini-cli
1
0
Fork 0
mirror of https://github.com/google-gemini/gemini-cli.git synced 2026-07-10 01:50:20 -07:00
Code Issues Packages Projects Releases Wiki Activity

Compression eval.

Browse Source
This commit is contained in:
Christian Gunderman
2026-04-02 09:44:19 -07:00
parent 704ad573f1
commit de1d5d3ca2
28 changed files with 17219 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
evals/data/compression/scenario-subtle-preference.json
+669
View File
@@ -0,0 +1,669 @@
{
"scenarioId": "scenario-subtle-preference",
"description": "Tests recall of subtle stylistic preferences.",
"history": [
{
"role": "user",
"parts": [
{
"text": "Whenever you write a commit message, always prefix it with [EVAL-TEST]."
}
]
},
{
"role": "model",
"parts": [
{
"text": "I will use the [EVAL-TEST] prefix."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 0"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 0 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 1"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 1 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 2"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 2 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 3"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 3 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 4"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 4 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 5"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 5 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 6"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 6 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 7"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 7 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 8"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 8 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 9"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 9 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 10"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 10 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 11"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 11 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 12"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 12 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 13"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 13 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 14"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 14 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 15"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 15 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 16"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 16 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 17"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 17 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 18"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 18 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 19"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 19 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 20"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 20 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 21"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 21 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 22"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 22 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 23"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 23 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 24"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 24 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 25"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 25 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 26"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 26 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 27"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 27 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 28"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 28 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 29"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 29 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 30"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 30 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 31"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 31 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 32"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 32 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 33"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 33 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 34"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 34 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 35"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 35 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 36"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 36 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 37"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 37 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 38"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 38 modified."
}
]
},
{
"role": "user",
"parts": [
{
"text": "Modify file 39"
}
]
},
{
"role": "model",
"parts": [
{
"text": "File 39 modified."
}
]
}
],
"questions": [
{
"id": "commit-prefix",
"prompt": "What prefix should I use for all commit messages?",
"expectedSubstring": "[EVAL-TEST]"
}
]
}