gemini-cli/evals/data/compression/scenario-milestones.json

{
  "scenarioId": "scenario-milestones",
  "description": "Tests recall of user-provided code snippets or previous milestone summaries from many turns prior.",
  "history": [
    {
      "role": "user",
      "parts": [
        {
          "text": "Here is the critical tax function for reference:\n```typescript\nfunction calculate_corporate_taxes(income) { return income * 0.21; }\n```\nKeep this in mind."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "I have memorized the calculate_corporate_taxes function."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 0..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 0 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 1..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 1 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 2..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 2 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 3..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 3 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 4..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 4 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 5..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 5 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 6..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 6 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 7..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 7 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 8..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 8 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 9..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 9 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 10..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 10 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 11..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 11 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 12..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 12 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 13..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 13 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 14..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 14 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Summarize progress so far."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "We have debugged for 15 iterations. We have achieved milestone-alpha-achieved. Still working on the core logic."
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 15..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 15 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 16..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 16 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 17..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 17 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 18..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 18 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 19..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 19 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 20..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 20 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 21..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 21 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 22..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 22 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 23..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 23 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 24..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 24 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 25..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 25 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 26..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 26 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 27..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 27 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 28..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 28 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    },
    {
      "role": "user",
      "parts": [
        {
          "text": "Debug iteration 29..."
        }
      ]
    },
    {
      "role": "model",
      "parts": [
        {
          "text": "Iteration 29 failed. Error: Error: Cannot read properties of undefined (reading \"config\")\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Module.load (node:internal/modules/cjs/loader:1207:32)\n    at Function.Module._load (node:internal/modules/cjs/loader:1023:12)\n    at Module._compile (node:internal/modules/cjs/loader:1376:14)\n    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1435:10)\n    at Modul"
        }
      ]
    }
  ],
  "questions": [
    {
      "id": "tax-function-name",
      "prompt": "What was the exact name of the tax function I provided earlier?",
      "expectedSubstring": "calculate_corporate_taxes"
    },
    {
      "id": "milestone-name",
      "prompt": "What was the exact name of the milestone we achieved earlier?",
      "expectedSubstring": "milestone-alpha-achieved"
    }
  ]
}