feat(optimizer): establish foundation with actions and deterministic metrics

.github/workflows/optimizer-brain.yml

@@ -0,0 +1,48 @@
+name: Optimizer Brain
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  issues: write
+  pull-requests: write
+
+jobs:
+  brain:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Run Optimizer Brain
+        run: |
+          cd tools/optimizer
+          npx tsx index.ts --investigate
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+          EXECUTE_ACTIONS: 'true'
+
+      - name: Upload Brain Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: optimizer-brain-results
+          path: |
+            metrics-before.csv
+            metrics-after.csv
+            *-before.csv
+            *-after.csv
+            investigations/INVESTIGATIONS.md
+            lessons-learned.md

.github/workflows/optimizer-pulse.yml

@@ -0,0 +1,45 @@
+name: Optimizer Pulse
+
+on:
+  schedule:
+    - cron: '*/30 * * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  issues: write
+  pull-requests: write
+
+jobs:
+  pulse:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Run Optimizer Pulse
+        run: |
+          cd tools/optimizer
+          npx tsx index.ts --pulse
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          EXECUTE_ACTIONS: 'true'
+
+      - name: Upload Metrics Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: optimizer-pulse-results
+          path: |
+            metrics-before.csv
+            metrics-after.csv
+            *-before.csv
+            *-after.csv

tools/optimizer/index.ts

@@ -9,9 +9,10 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
 async function main() {
   const program = new Command();
   program
-    .option('--investigate', 'Run investigation and process-updater phase', false)
+    .option('--investigate', 'Run deep investigation (Brain phase)', false)
+    .option('--pulse', 'Run high-frequency reflex actions (Pulse phase)', false)
     .option('--create-pr', 'Create a PR when updating processes', false)
-    .option('--execute-actions', 'Actually execute destructive or state-changing actions (e.g., closing issues, commenting)', false)
+    .option('--execute-actions', 'Actually execute state-changing actions', false)
     .parse(process.argv);
 
   const options = program.opts();
@@ -21,73 +22,118 @@ async function main() {
 
   const rootDir = path.resolve(__dirname, '../..');
   
-  // Ensure history directory exists so agent doesn't fail listing it
+  // Ensure history directory exists
   await fs.mkdir(path.join(rootDir, 'history'), { recursive: true });
 
-  // 0. Fetch previous artifacts
-  try {
-    console.log('Checking for previous artifacts...');
-    // Check if any run exists for the current branch
-    const branch = execSync('git rev-parse --abbrev-ref HEAD', { encoding: 'utf-8' }).trim();
-    const runCheck = execSync(`gh run list --branch ${branch} --limit 1 --json databaseId --jq '.[0].databaseId' || true`, { encoding: 'utf-8' }).trim();
-    
-    if (runCheck && runCheck !== '') {
-      console.log('Attempting to fetch previous artifacts into history/ (timeout 30s)...');
-      await fs.mkdir(path.join(rootDir, 'history'), { recursive: true });
-      // Download will fail gracefully if the artifact name doesn't match
-      execSync(`gh run download --name optimizer-results --pattern "*.csv" --dir history > /dev/null 2>&1 || true`, { 
-        stdio: 'inherit',
-        timeout: 30000,
-        cwd: rootDir
-      });
-    } else {
-      console.log('No previous runs found, skipping download.');
-    }
-  } catch (err) {
-    console.warn('Artifact check/download skipped, proceeding with fresh state.');
-  }
+  // 0. Fetch previous artifacts (Memory)
+  await syncHistory(rootDir);
 
   const policyPath = options.executeActions ? undefined : path.join(__dirname, 'policies', 'readonly-gh.toml');
 
-  // 1. Initial Metrics
-  await runPhase('metrics', { PRE_RUN: 'true' }, options, policyPath);
+  // 1. Initial Metrics (Deterministic)
+  await runMetrics(true, rootDir);
 
-  // 2. Investigation & Update Processes (Optional)
+  // 2. Investigation & Update Processes (Agentic - Brain Phase)
   if (options.investigate) {
-    await runPhase('investigations', {
+    await runAgentPhase('investigations', {
       EXECUTE_ACTIONS: String(options.executeActions),
     }, options, undefined);
 
     // 3. Critique Phase (Only runs if investigations ran)
-    await runPhase('critique', {
+    await runAgentPhase('critique', {
       CREATE_PR: String(options.createPr),
       EXECUTE_ACTIONS: String(options.executeActions),
     }, options, undefined);
   }
 
-  // 4. Run Processes
-  await runPhase('processes', {
-    EXECUTE_ACTIONS: String(options.executeActions),
-  }, options, policyPath);
+  // 4. Run Processes (Pulse Phase)
+  // In v1, Pulse runs are deterministic script executions.
+  if (options.pulse || options.investigate) {
+    await runProcesses(options.executeActions, rootDir);
+  }
 
-  // 5. Final Metrics
-  await runPhase('metrics', { PRE_RUN: 'false' }, options, policyPath);
+  // 5. Final Metrics (Deterministic)
+  await runMetrics(false, rootDir);
 
   console.log('\nOptimizer1000 completed.');
 }
 
-async function runPhase(phaseDir: string, env: Record<string, string>, options: any, policyPath?: string): Promise<string | undefined> {
-  console.log(`\n--- Phase: ${phaseDir} ---`);
+/**
+ * Runs repository metrics deterministically by executing scripts in metrics/scripts/
+ */
+async function runMetrics(preRun: boolean, rootDir: string) {
+  const phase = preRun ? 'before' : 'after';
+  console.log(`\n--- Phase: metrics (${phase}) ---`);
+  
+  const scriptsDir = path.join(__dirname, 'metrics', 'scripts');
+  const scripts = await fs.readdir(scriptsDir);
+  const jsScripts = scripts.filter(s => s.endsWith('.js') || s.endsWith('.ts'));
+  
+  const results: any[] = [];
+  
+  for (const script of jsScripts) {
+    console.log(`Running metric script: ${script}`);
+    try {
+      const scriptPath = path.join(scriptsDir, script);
+      const command = script.endsWith('.ts') ? `npx tsx ${scriptPath}` : `node ${scriptPath}`;
+      const output = execSync(command, { encoding: 'utf-8', cwd: rootDir });
+      
+      // Scripts should output JSON objects per line
+      const lines = output.trim().split('\n');
+      for (const line of lines) {
+        try {
+          results.push(JSON.parse(line));
+        } catch {
+          // Fallback for non-JSON output
+          if (line) console.log(`[Script Output]: ${line}`);
+        }
+      }
+    } catch (err: any) {
+      console.error(`Error running ${script}:`, err.message);
+    }
+  }
+  
+  const outputFile = path.join(rootDir, `metrics-${phase}.csv`);
+  const csvContent = jsonToCsv(results);
+  await fs.writeFile(outputFile, csvContent);
+  console.log(`Metrics saved to ${outputFile}`);
+}
+
+/**
+ * Runs optimization processes deterministically
+ */
+async function runProcesses(execute: boolean, rootDir: string) {
+  console.log(`\n--- Phase: processes ---`);
+  const scriptsDir = path.join(__dirname, 'processes', 'scripts');
+  
+  // We look for a PROCESSES.md to see what's active, but for now we just run all .ts scripts in the dir
+  const scripts = await fs.readdir(scriptsDir);
+  const activeScripts = scripts.filter(s => s.endsWith('.ts') && s !== 'utils.ts');
+  
+  for (const script of activeScripts) {
+    console.log(`Running process: ${script}`);
+    try {
+      const scriptPath = path.join(scriptsDir, script);
+      execSync(`npx tsx ${scriptPath}`, {
+        stdio: 'inherit',
+        cwd: rootDir,
+        env: { ...process.env, EXECUTE_ACTIONS: String(execute) }
+      });
+    } catch (err: any) {
+      console.error(`Error running process ${script}:`, err.message);
+    }
+  }
+}
+
+/**
+ * Runs an agentic phase using GCLI
+ */
+async function runAgentPhase(phaseDir: string, env: Record<string, string>, options: any, policyPath?: string) {
+  console.log(`\n--- Phase: ${phaseDir} (Agentic) ---`);
   const phasePath = path.join(__dirname, phaseDir);
   
-  let promptFile: string | undefined;
-  try {
-    const files = await fs.readdir(phasePath);
-    promptFile = files.find(f => f.endsWith('-AGENT.md'));
-  } catch (err) {
-    console.warn(`Directory ${phaseDir} not found or inaccessible.`);
-    return;
-  }
+  const files = await fs.readdir(phasePath);
+  const promptFile = files.find(f => f.endsWith('-AGENT.md'));
   
   if (!promptFile) {
     console.warn(`No agent prompt found in ${phaseDir}`);
@@ -98,47 +144,62 @@ async function runPhase(phaseDir: string, env: Record<string, string>, options:
   const instructionsContent = await fs.readFile(instructionsPath, 'utf8');
 
   const envString = Object.entries(env).map(([k, v]) => `${k}=${v}`).join('\n');
-  const userPrompt = `Execution Context:\n${envString}\n\n${instructionsContent}\n\nPlease proceed with the ${phaseDir} tasks as defined in your instructions. Always output CSV files as requested.`;
+  const userPrompt = `Execution Context:\n${envString}\n\n${instructionsContent}\n\nPlease proceed with the ${phaseDir} tasks.`;
 
-  console.log(`Running agent with prompt: ${promptFile}`);
-  
-  // Resolve root to call the CLI binary
   const rootDir = path.resolve(__dirname, '../..');
+  const cliPath = path.join(rootDir, 'packages', 'cli');
+  const args = ['--prompt', userPrompt, '--yolo', '--model', 'gemini-3-flash-preview'];
   
-  try {
-    // Run GCLI non-interactively. Use --yolo to auto-approve 'allow' rules,
-    // but policies can still 'deny' actions.
-    const cliPath = path.join(rootDir, 'packages', 'cli');
-    const args = ['--prompt', userPrompt, '--yolo', '--model', 'gemini-3-flash-preview'];
-    
-    if (policyPath) {
-      args.push('--admin-policy', policyPath);
-    }
-    
-    await new Promise<void>((resolve, reject) => {
-      const child = spawn('node', [cliPath, ...args], {
-        stdio: 'inherit',
-        cwd: rootDir,
-        env: { ...process.env, ...env }
-      });
-
-      child.on('close', (code) => {
-        if (code === 0) {
-          resolve();
-        } else {
-          reject(new Error(`Exit code ${code}`));
-        }
-      });
-
-      child.on('error', (err) => {
-        reject(err);
-      });
+  if (policyPath) args.push('--admin-policy', policyPath);
+  
+  await new Promise<void>((resolve, reject) => {
+    const child = spawn('node', [cliPath, ...args], {
+      stdio: 'inherit',
+      cwd: rootDir,
+      env: { ...process.env, ...env }
     });
-  } catch (err: any) {
-    console.error(`Error in phase ${phaseDir}:`, err.message);
-  }
+    child.on('close', code => code === 0 ? resolve() : reject(new Error(`Exit code ${code}`)));
+    child.on('error', reject);
+  });
+}
 
-  console.log(`\n--- Finished Phase: ${phaseDir} ---`);
+async function syncHistory(rootDir: string) {
+  try {
+    console.log('Checking for previous artifacts...');
+    const branch = execSync('git rev-parse --abbrev-ref HEAD', { encoding: 'utf-8' }).trim();
+    const runCheck = execSync(`gh run list --branch ${branch} --limit 1 --json databaseId --jq '.[0].databaseId' || true`, { encoding: 'utf-8' }).trim();
+    
+    if (runCheck && runCheck !== '') {
+      console.log('Fetching previous artifacts into history/...');
+      execSync(`gh run download --name optimizer-pulse-results --pattern "*.csv" --dir history > /dev/null 2>&1 || true`, { 
+        stdio: 'inherit',
+        timeout: 30000,
+        cwd: rootDir
+      });
+      execSync(`gh run download --name optimizer-brain-results --pattern "*.csv" --dir history > /dev/null 2>&1 || true`, { 
+        stdio: 'inherit',
+        timeout: 30000,
+        cwd: rootDir
+      });
+    }
+  } catch (err) {
+    console.warn('Artifact sync skipped.');
+  }
+}
+
+function jsonToCsv(items: any[]): string {
+  if (items.length === 0) return '';
+  const headers = Array.from(new Set(items.flatMap(item => Object.keys(item))));
+  const csvRows = [headers.join(',')];
+  for (const item of items) {
+    const values = headers.map(header => {
+      const val = item[header] ?? '';
+      const stringVal = String(val);
+      return stringVal.includes(',') ? `"${stringVal.replace(/"/g, '""')}"` : stringVal;
+    });
+    csvRows.push(values.join(','));
+  }
+  return csvRows.join('\n');
 }
 
 main().catch(err => {

tools/optimizer/metrics/METRICS-AGENT.md

@@ -1,9 +0,0 @@
-# Metrics Agent
-
-Your task is to gather repository metrics.
-
-1. Check for historical data in the `history/` directory to understand previous trends.
-2. Run all scripts in the `metrics/scripts/` directory.
-3. Output the results to a `metrics-before.csv` file in the project root if this is the start of the run (determined by the presence of `PRE_RUN=true`), or `metrics-after.csv` in the root if it is the end (`PRE_RUN=false`).
-4. For any targeted repository concept (e.g., issues), generate a `[concept]-before.csv` (or `-after.csv`) in the project root listing the items and their current state.
-5. If a tool fails (e.g., policy denial or script error), report the exact error and DO NOT claim success for that specific task. Attempt to proceed with other scripts if possible.