gemini-cli/evals/data/compression/scenario-context-amnesia.json

{
  "scenarioId": "scenario-context-amnesia",
  "description": "Tests if the agent remembers the content of a file it recently read after compression. Addresses \"Context Amnesia\" telemetry findings.",
  "history": [
    {
      "role": "user",
      "parts": [
        {
          "text": "Please read src/config/app.config.json to check the timeout settings."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "functionCall": {
            "name": "read_file",
            "args": {
              "file_path": "src/config/app.config.json"
            }
          }
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "functionResponse": {
            "name": "read_file",
            "response": {
              "content": "{\n  \"timeout\": 4500,\n  \"retries\": 3,\n  \"environment\": \"staging\"\n}"
            }
          }
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "I see the timeout is 4500ms. Now let's look at the logs."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 0..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 0 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 1..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 1 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 2..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 2 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 3..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 3 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 4..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 4 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 5..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 5 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 6..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 6 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 7..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 7 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 8..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 8 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 9..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 9 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 10..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 10 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 11..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 11 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 12..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 12 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 13..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 13 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 14..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 14 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 15..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 15 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 16..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 16 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 17..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 17 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 18..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 18 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 19..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 19 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 20..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 20 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 21..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 21 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 22..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 22 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 23..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 23 analyzed. No errors found."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking system log part 24..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Log part 24 analyzed. No errors found."
        }
      ]
    }
  ],
  "questions": [
    {
      "id": "recall-file-content",
      "prompt": "What was the exact value of \"timeout\" in src/config/app.config.json?",
      "expectedSubstring": "4500"
    }
  ]
}