gemini-cli/evals/data/compression/scenario-replace-loop.json

{
  "scenarioId": "scenario-replace-loop",
  "description": "Tests if the agent retains the EXACT string content needed for the \"replace\" tool after compression. Targets the \"Read-Search-Replace-Fail\" loop.",
  "history": [
    {
      "role": "user",
      "parts": [
        {
          "text": "Find the session validation logic in session.ts."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "functionCall": {
            "name": "grep_search",
            "args": {
              "pattern": "session.isValid"
            }
          }
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "functionResponse": {
            "name": "grep_search",
            "response": {
              "results": [
                {
                  "file": "src/auth/session.ts",
                  "line": 42,
                  "content": "if (session.isValid() && !session.isExpired(Date.now())) {"
                }
              ]
            }
          }
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Found it on line 42: if (session.isValid() && !session.isExpired(Date.now())) {"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 0.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 0.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 1.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 1.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 2.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 2.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 3.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 3.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 4.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 4.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 5.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 5.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 6.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 6.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 7.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 7.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 8.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 8.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 9.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 9.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 10.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 10.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 11.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 11.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 12.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 12.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 13.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 13.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 14.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 14.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 15.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 15.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 16.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 16.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 17.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 17.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 18.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 18.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 19.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 19.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 20.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 20.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 21.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 21.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 22.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 22.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 23.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 23.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 24.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 24.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 25.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 25.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 26.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 26.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 27.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 27.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 28.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 28.ts is unrelated."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Checking unrelated file 29.ts"
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "File 29.ts is unrelated."
        }
      ]
    }
  ],
  "questions": [
    {
      "id": "recall-exact-line",
      "prompt": "I need to use the \"replace\" tool on src/auth/session.ts. What is the EXACT literal string currently on line 42 that I should use as \"old_string\"?",
      "expectedSubstring": "if (session.isValid() && !session.isExpired(Date.now())) {"
    }
  ]
}