gemini-cli/evals/data/compression/scenario-subtle-preference.json

{
  "scenarioId": "scenario-subtle-preference",
  "description": "Tests recall of subtle stylistic preferences.",
  "history": [
    {
      "role": "user",
      "parts": [
        {
          "text": "Whenever you write a commit message, always prefix it with [EVAL-TEST]."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "I will use the [EVAL-TEST] prefix."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 0"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 0 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 1"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 1 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 2"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 2 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 3"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 3 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 4"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 4 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 5"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 5 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 6"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 6 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 7"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 7 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 8"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 8 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 9"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 9 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 10"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 10 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 11"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 11 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 12"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 12 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 13"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 13 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 14"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 14 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 15"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 15 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 16"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 16 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 17"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 17 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 18"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 18 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 19"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 19 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 20"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 20 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 21"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 21 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 22"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 22 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 23"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 23 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 24"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 24 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 25"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 25 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 26"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 26 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 27"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 27 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 28"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 28 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 29"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 29 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 30"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 30 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 31"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 31 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 32"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 32 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 33"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 33 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 34"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 34 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 35"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 35 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 36"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 36 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 37"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 37 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 38"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 38 modified."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Modify file 39"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 39 modified."
        }
      ]
    }
  ],
  "questions": [
    {
      "id": "commit-prefix",
      "prompt": "What prefix should I use for all commit messages?",
      "expectedSubstring": "[EVAL-TEST]"
    }
  ]
}