gemini-cli/evals/data/compression/scenario-surgical-imprecision.json

{
  "scenarioId": "scenario-surgical-imprecision",
  "description": "Tests if the agent can perform a surgical, single-line edit after compression. Targets \"Over-Replacement\".",
  "history": [
    {
      "role": "user",
      "parts": [
        {
          "text": "Read src/main.ts."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "functionCall": {
            "name": "read_file",
            "args": {
              "file_path": "src/main.ts"
            }
          }
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "functionResponse": {
            "name": "read_file",
            "response": {
              "content": "export function initializeApp(config: AppConfig) {\n  const logger = new Logger(config.logLevel);\n  const db = new Database(config.dbUrl);\n  const server = new Server(config.port, config.host);\n  return { logger, db, server };\n}"
            }
          }
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "I have read the initializeApp function."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 0..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 0 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 1..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 1 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 2..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 2 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 3..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 3 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 4..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 4 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 5..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 5 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 6..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 6 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 7..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 7 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 8..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 8 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 9..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 9 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 10..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 10 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 11..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 11 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 12..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 12 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 13..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 13 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 14..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 14 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 15..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 15 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 16..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 16 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 17..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 17 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 18..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 18 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 19..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 19 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 20..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 20 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 21..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 21 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 22..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 22 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 23..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 23 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 24..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 24 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 25..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 25 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 26..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 26 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 27..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 27 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 28..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 28 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 29..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 29 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 30..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 30 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 31..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 31 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 32..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 32 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 33..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 33 passed."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Perform unrelated environment check 34..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Env check 34 passed."
        }
      ]
    }
  ],
  "questions": [
    {
      "id": "surgical-edit-check",
      "prompt": "I want to change ONLY the \"config.dbUrl\" argument to \"config.connectionString\" in the initializeApp function. What is the MINIMAL, single-line \"old_string\" I should use for a surgical replace?",
      "expectedSubstring": "new Database(config.dbUrl)"
    }
  ]
}