gemini-cli/evals/data/compression/scenario-myopic-keyhole.json

{
  "scenarioId": "scenario-myopic-keyhole",
  "description": "Tests if the agent maintains a macro-understanding of file structure when reading in small chunks. Targets the \"Loss of Macro-Context\".",
  "history": [
    {
      "role": "user",
      "parts": [
        {
          "text": "Investigate the Store class in src/store/base.ts."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "functionCall": {
            "name": "read_file",
            "args": {
              "file_path": "src/store/base.ts",
              "start_line": 1,
              "end_line": 10
            }
          }
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "functionResponse": {
            "name": "read_file",
            "response": {
              "content": "import { BaseStore } from \"./types\";\n\n/**\n * Global application store.\n */"
            }
          }
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "functionCall": {
            "name": "read_file",
            "args": {
              "file_path": "src/store/base.ts",
              "start_line": 11,
              "end_line": 20
            }
          }
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "functionResponse": {
            "name": "read_file",
            "response": {
              "content": "export class Store extends BaseStore {\n  private state: any;\n  constructor() {"
            }
          }
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "functionCall": {
            "name": "read_file",
            "args": {
              "file_path": "src/store/base.ts",
              "start_line": 21,
              "end_line": 30
            }
          }
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "functionResponse": {
            "name": "read_file",
            "response": {
              "content": "    super();\n    this.state = {};\n  }"
            }
          }
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "I have read the Store class definition in chunks."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 0"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 0 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 1"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 1 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 2"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 2 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 3"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 3 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 4"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 4 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 5"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 5 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 6"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 6 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 7"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 7 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 8"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 8 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 9"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 9 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 10"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 10 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 11"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 11 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 12"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 12 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 13"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 13 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 14"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 14 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 15"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 15 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 16"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 16 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 17"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 17 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 18"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 18 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 19"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 19 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 20"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 20 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 21"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 21 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 22"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 22 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 23"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 23 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 24"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 24 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 25"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 25 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 26"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 26 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 27"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 27 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 28"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 28 complete."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated check 29"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Check 29 complete."
        }
      ]
    }
  ],
  "questions": [
    {
      "id": "macro-context-inheritance",
      "prompt": "What class does the \"Store\" class in src/store/base.ts inherit from?",
      "expectedSubstring": "BaseStore"
    }
  ]
}