mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-04-21 02:24:09 -07:00
feat(offload): implement WorkerProvider abstraction and robust IAP fallback
This commit is contained in:
@@ -16,7 +16,10 @@
|
||||
- **Runtime**: The container runs as a persistent service (`--restart always`) acting as a "Remote Workstation" rather than an ephemeral task.
|
||||
|
||||
## Orchestration Logic
|
||||
- **Fast-Path SSH**: Land on the VM Host via standard SSH (using an alias like `gcli-worker`).
|
||||
- **Worker Provider Abstraction**: Infrastructure is managed via a `WorkerProvider` interface (e.g., `GceCosProvider`). This decouples the orchestration logic from the underlying platform.
|
||||
- **Robust Connectivity**: The system uses a dual-path connectivity strategy:
|
||||
1. **Fast-Path SSH**: Primary connection via a standard SSH alias (`gcli-worker`) for high-performance synchronization and interaction.
|
||||
2. **IAP Fallback**: Automatic fallback to `gcloud compute ssh --tunnel-through-iap` for users off-VPC or when direct DNS resolution fails.
|
||||
- **Context Execution**: Use `docker exec -it maintainer-worker ...` for interactive tasks and `tmux` sessions. This provides persistence against connection drops while keeping the host OS "invisible."
|
||||
- **Path Resolution**: Both Host and Container must share identical tilde (`~`) paths to avoid mapping confusion in automation scripts.
|
||||
|
||||
|
||||
@@ -1,17 +1,13 @@
|
||||
/**
|
||||
* Universal Deep Review Checker (Local)
|
||||
*
|
||||
* Polls the remote machine for task status.
|
||||
*/
|
||||
import { spawnSync } from 'child_process';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { ProviderFactory } from './providers/ProviderFactory.ts';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = path.resolve(__dirname, '../../../..');
|
||||
|
||||
export async function runChecker(args: string[]) {
|
||||
export async function runChecker(args: string[], env: NodeJS.ProcessEnv = process.env) {
|
||||
const prNumber = args[0];
|
||||
if (!prNumber) {
|
||||
console.error('Usage: npm run review:check <PR_NUMBER>');
|
||||
@@ -20,7 +16,7 @@ export async function runChecker(args: string[]) {
|
||||
|
||||
const settingsPath = path.join(REPO_ROOT, '.gemini/settings.json');
|
||||
if (!fs.existsSync(settingsPath)) {
|
||||
console.error('❌ Settings not found. Run "npm run review:setup" first.');
|
||||
console.error('❌ Settings not found. Run "npm run offload:setup" first.');
|
||||
return 1;
|
||||
}
|
||||
const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
|
||||
@@ -29,9 +25,11 @@ export async function runChecker(args: string[]) {
|
||||
console.error('❌ Deep Review configuration not found.');
|
||||
return 1;
|
||||
}
|
||||
const { remoteHost, remoteWorkDir } = config;
|
||||
const { projectId, zone, remoteWorkDir } = config;
|
||||
const targetVM = `gcli-offload-${env.USER || 'mattkorwel'}`;
|
||||
const provider = ProviderFactory.getProvider({ projectId, zone, instanceName: targetVM });
|
||||
|
||||
console.log(`🔍 Checking remote status for PR #${prNumber} on ${remoteHost}...`);
|
||||
console.log(`🔍 Checking remote status for PR #${prNumber} on ${targetVM}...`);
|
||||
|
||||
const branchView = spawnSync('gh', ['pr', 'view', prNumber, '--json', 'headRefName', '-q', '.headRefName'], { shell: true });
|
||||
const branchName = branchView.stdout.toString().trim();
|
||||
@@ -42,13 +40,15 @@ export async function runChecker(args: string[]) {
|
||||
|
||||
console.log('\n--- Task Status ---');
|
||||
for (const task of tasks) {
|
||||
const checkExit = spawnSync('ssh', [remoteHost, `cat ${logDir}/${task}.exit 2>/dev/null`], { shell: true });
|
||||
if (checkExit.status === 0) {
|
||||
const code = checkExit.stdout.toString().trim();
|
||||
const exitFile = `${logDir}/${task}.exit`;
|
||||
const checkExit = await provider.getExecOutput(`[ -f ${exitFile} ] && cat ${exitFile}`, { wrapContainer: 'maintainer-worker' });
|
||||
|
||||
if (checkExit.status === 0 && checkExit.stdout.trim()) {
|
||||
const code = checkExit.stdout.trim();
|
||||
console.log(` ${code === '0' ? '✅' : '❌'} ${task.padEnd(10)}: ${code === '0' ? 'SUCCESS' : `FAILED (exit ${code})`}`);
|
||||
} else {
|
||||
const checkRunning = spawnSync('ssh', [remoteHost, `[ -f ${logDir}/${task}.log ]`], { shell: true });
|
||||
if (checkRunning.status === 0) {
|
||||
const checkRunning = await provider.exec(`[ -f ${logDir}/${task}.log ]`, { wrapContainer: 'maintainer-worker' });
|
||||
if (checkRunning === 0) {
|
||||
console.log(` ⏳ ${task.padEnd(10)}: RUNNING`);
|
||||
} else {
|
||||
console.log(` 💤 ${task.padEnd(10)}: PENDING`);
|
||||
|
||||
@@ -6,150 +6,84 @@
|
||||
import { spawnSync } from 'child_process';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { ProviderFactory } from './providers/ProviderFactory.ts';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = path.resolve(__dirname, '../../../..');
|
||||
|
||||
const PROJECT_ID = 'gemini-cli-team-quota';
|
||||
const USER = process.env.USER || 'mattkorwel';
|
||||
const INSTANCE_PREFIX = `gcli-offload-${USER}`;
|
||||
const DEFAULT_ZONE = 'us-west1-a';
|
||||
|
||||
async function listWorkers() {
|
||||
console.log(`🔍 Listing Offload Workers for ${USER} in ${PROJECT_ID}...`);
|
||||
|
||||
const result = spawnSync('gcloud', [
|
||||
spawnSync('gcloud', [
|
||||
'compute', 'instances', 'list',
|
||||
'--project', PROJECT_ID,
|
||||
'--filter', `name~^${INSTANCE_PREFIX}`,
|
||||
'--format', 'table(name,zone,status,networkInterfaces[0].networkIP:label=INTERNAL_IP,creationTimestamp)'
|
||||
], { stdio: 'inherit' });
|
||||
|
||||
if (result.status !== 0) {
|
||||
console.error('\n❌ Failed to list workers. Ensure you have access to the project and gcloud is authenticated.');
|
||||
}
|
||||
}
|
||||
|
||||
async function provisionWorker() {
|
||||
const name = INSTANCE_PREFIX;
|
||||
const zone = 'us-west1-a';
|
||||
const imageUri = 'us-docker.pkg.dev/gemini-code-dev/gemini-cli/maintainer:latest';
|
||||
|
||||
console.log(`🔍 Checking if worker ${name} already exists...`);
|
||||
const existCheck = spawnSync('gcloud', [
|
||||
'compute', 'instances', 'describe', name,
|
||||
'--project', PROJECT_ID,
|
||||
'--zone', zone
|
||||
], { stdio: 'pipe' });
|
||||
const provider = ProviderFactory.getProvider({
|
||||
projectId: PROJECT_ID,
|
||||
zone: DEFAULT_ZONE,
|
||||
instanceName: INSTANCE_PREFIX
|
||||
});
|
||||
|
||||
if (existCheck.status === 0) {
|
||||
console.log(`✅ Worker ${name} already exists and is ready for use.`);
|
||||
const status = await provider.getStatus();
|
||||
if (status.status !== 'UNKNOWN' && status.status !== 'ERROR') {
|
||||
console.log(`✅ Worker ${INSTANCE_PREFIX} already exists and is ${status.status}.`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`🚀 Provisioning modern container worker (COS + Startup Script): ${name}...`);
|
||||
|
||||
// Get local public key for native SSH access
|
||||
const pubKeyPath = path.join(os.homedir(), '.ssh/google_compute_engine.pub');
|
||||
const pubKey = fs.existsSync(pubKeyPath) ? fs.readFileSync(pubKeyPath, 'utf8').trim() : '';
|
||||
const sshKeyMetadata = pubKey ? `${USER}:${pubKey}` : '';
|
||||
|
||||
// Direct Startup Script for COS (Native Docker launch)
|
||||
const startupScript = `#!/bin/bash
|
||||
# Pull and Run the maintainer container
|
||||
docker pull ${imageUri}
|
||||
docker run -d --name maintainer-worker --restart always \\
|
||||
-v /home/node/dev:/home/node/dev:rw \\
|
||||
-v /home/node/.gemini:/home/node/.gemini:rw \\
|
||||
-v /home/node/.offload:/home/node/.offload:rw \\
|
||||
${imageUri} /bin/bash -c "while true; do sleep 1000; done"
|
||||
`;
|
||||
|
||||
const result = spawnSync('gcloud', [
|
||||
'compute', 'instances', 'create', name,
|
||||
'--project', PROJECT_ID,
|
||||
'--zone', 'us-west1-a',
|
||||
'--machine-type', 'n2-standard-8',
|
||||
'--image-family', 'cos-stable',
|
||||
'--image-project', 'cos-cloud',
|
||||
'--boot-disk-size', '200GB',
|
||||
'--boot-disk-type', 'pd-balanced',
|
||||
'--metadata', `startup-script=${startupScript},enable-oslogin=TRUE${sshKeyMetadata ? `,ssh-keys=${sshKeyMetadata}` : ''}`,
|
||||
'--labels', `owner=${USER.replace(/[^a-z0-9_-]/g, '_')},type=offload-worker`,
|
||||
'--tags', `gcli-offload-${USER}`,
|
||||
'--network-interface', 'network-tier=PREMIUM,no-address',
|
||||
'--scopes', 'https://www.googleapis.com/auth/cloud-platform'
|
||||
], { stdio: 'inherit' });
|
||||
|
||||
if (result.status === 0) {
|
||||
console.log(`\n✅ Worker ${name} is being provisioned.`);
|
||||
console.log(`👉 Container 'maintainer-worker' will start natively via Cloud-Init.`);
|
||||
}
|
||||
}
|
||||
|
||||
async function createImage() {
|
||||
const name = `gcli-maintainer-worker-build-${Math.floor(Date.now() / 1000)}`;
|
||||
const zone = 'us-west1-a';
|
||||
const imageName = 'gcli-maintainer-worker-v1';
|
||||
|
||||
console.log(`🏗️ Building Maintainer Image: ${imageName}...`);
|
||||
|
||||
// 1. Create a temporary builder VM
|
||||
console.log(' - Creating temporary builder VM...');
|
||||
spawnSync('gcloud', [
|
||||
'compute', 'instances', 'create', name,
|
||||
'--project', PROJECT_ID,
|
||||
'--zone', zone,
|
||||
'--machine-type', 'n2-standard-4',
|
||||
'--image-family', 'ubuntu-2204-lts',
|
||||
'--image-project', 'ubuntu-os-cloud',
|
||||
'--metadata-from-file', `startup-script=.gemini/skills/offload/scripts/provision-worker.sh`
|
||||
], { stdio: 'inherit' });
|
||||
|
||||
console.log('\n⏳ Waiting for provisioning to complete (this takes ~3-5 mins)...');
|
||||
console.log(' - You can tail the startup script via:');
|
||||
console.log(` gcloud compute instances get-serial-port-output ${name} --project ${PROJECT_ID} --zone ${zone} --follow`);
|
||||
|
||||
// Note: For a true automation we'd poll here, but for a maintainer tool,
|
||||
// we'll provide the instructions to finalize.
|
||||
console.log(`\n👉 Once provisioning is DONE, run these commands to finalize:`);
|
||||
console.log(` 1. gcloud compute instances stop ${name} --project ${PROJECT_ID} --zone ${zone}`);
|
||||
console.log(` 2. gcloud compute images create ${imageName} --project ${PROJECT_ID} --source-disk ${name} --source-disk-zone ${zone} --family gcli-maintainer-worker`);
|
||||
console.log(` 3. gcloud compute instances delete ${name} --project ${PROJECT_ID} --zone ${zone} --quiet`);
|
||||
await provider.provision();
|
||||
}
|
||||
|
||||
async function stopWorker() {
|
||||
const name = INSTANCE_PREFIX;
|
||||
const zone = 'us-west1-a';
|
||||
const provider = ProviderFactory.getProvider({
|
||||
projectId: PROJECT_ID,
|
||||
zone: DEFAULT_ZONE,
|
||||
instanceName: INSTANCE_PREFIX
|
||||
});
|
||||
|
||||
console.log(`🛑 Stopping offload worker: ${name}...`);
|
||||
const result = spawnSync('gcloud', [
|
||||
'compute', 'instances', 'stop', name,
|
||||
'--project', PROJECT_ID,
|
||||
'--zone', zone
|
||||
], { stdio: 'inherit' });
|
||||
|
||||
if (result.status === 0) {
|
||||
console.log(`\n✅ Worker ${name} has been stopped.`);
|
||||
}
|
||||
console.log(`🛑 Stopping offload worker: ${INSTANCE_PREFIX}...`);
|
||||
await provider.stop();
|
||||
}
|
||||
|
||||
async function remoteStatus() {
|
||||
const name = INSTANCE_PREFIX;
|
||||
const sshConfigPath = path.join(path.dirname(fileURLToPath(import.meta.url)), '../../offload_ssh_config');
|
||||
console.log(`📡 Fetching remote status from ${name}...`);
|
||||
spawnSync('ssh', ['-F', sshConfigPath, 'gcli-worker', 'tsx .offload/scripts/status.ts'], { stdio: 'inherit', shell: true });
|
||||
const settingsPath = path.join(REPO_ROOT, '.gemini/settings.json');
|
||||
if (!fs.existsSync(settingsPath)) {
|
||||
console.error('❌ Settings not found. Run "npm run offload:setup" first.');
|
||||
return;
|
||||
}
|
||||
const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
|
||||
const config = settings.maintainer?.deepReview;
|
||||
|
||||
const provider = ProviderFactory.getProvider({
|
||||
projectId: config?.projectId || PROJECT_ID,
|
||||
zone: config?.zone || DEFAULT_ZONE,
|
||||
instanceName: INSTANCE_PREFIX
|
||||
});
|
||||
|
||||
console.log(`📡 Fetching remote status from ${INSTANCE_PREFIX}...`);
|
||||
await provider.exec('tsx .offload/scripts/status.ts');
|
||||
}
|
||||
|
||||
async function rebuildWorker() {
|
||||
const name = INSTANCE_PREFIX;
|
||||
console.log(`🔥 Rebuilding worker ${name}...`);
|
||||
console.log(`🔥 Rebuilding worker ${INSTANCE_PREFIX}...`);
|
||||
|
||||
// Clear isolated known_hosts to prevent ID mismatch on rebuild
|
||||
const knownHostsPath = path.join(path.dirname(fileURLToPath(import.meta.url)), '../../offload_known_hosts');
|
||||
const knownHostsPath = path.join(REPO_ROOT, '.gemini/offload_known_hosts');
|
||||
if (fs.existsSync(knownHostsPath)) {
|
||||
console.log(` - Clearing isolated known_hosts...`);
|
||||
fs.unlinkSync(knownHostsPath);
|
||||
}
|
||||
|
||||
spawnSync('gcloud', ['compute', 'instances', 'delete', name, '--project', PROJECT_ID, '--zone', 'us-west1-a', '--quiet'], { stdio: 'inherit' });
|
||||
spawnSync('gcloud', ['compute', 'instances', 'delete', INSTANCE_PREFIX, '--project', PROJECT_ID, '--zone', DEFAULT_ZONE, '--quiet'], { stdio: 'inherit' });
|
||||
await provisionWorker();
|
||||
}
|
||||
|
||||
@@ -172,9 +106,6 @@ async function main() {
|
||||
case 'status':
|
||||
await remoteStatus();
|
||||
break;
|
||||
case 'create-image':
|
||||
await createImage();
|
||||
break;
|
||||
default:
|
||||
console.error(`❌ Unknown fleet action: ${action}`);
|
||||
process.exit(1);
|
||||
|
||||
@@ -3,10 +3,10 @@
|
||||
*
|
||||
* Automatically connects to your dedicated worker and launches a persistent tmux task.
|
||||
*/
|
||||
import { spawnSync } from 'child_process';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { ProviderFactory } from './providers/ProviderFactory.ts';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = path.resolve(__dirname, '../../../..');
|
||||
@@ -24,38 +24,35 @@ export async function runOrchestrator(args: string[], env: NodeJS.ProcessEnv = p
|
||||
|
||||
// 1. Load Settings
|
||||
const settingsPath = path.join(REPO_ROOT, '.gemini/settings.json');
|
||||
const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
|
||||
const config = settings.maintainer?.deepReview;
|
||||
if (!config) {
|
||||
if (!fs.existsSync(settingsPath)) {
|
||||
console.error('❌ Settings not found. Run "npm run offload:setup" first.');
|
||||
return 1;
|
||||
}
|
||||
const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
|
||||
const config = settings.maintainer?.deepReview;
|
||||
if (!config) {
|
||||
console.error('❌ Deep Review configuration not found.');
|
||||
return 1;
|
||||
}
|
||||
|
||||
const { projectId, zone, remoteHost, remoteWorkDir } = config;
|
||||
const { projectId, zone, remoteWorkDir } = config;
|
||||
const targetVM = `gcli-offload-${env.USER || 'mattkorwel'}`;
|
||||
|
||||
const provider = ProviderFactory.getProvider({ projectId, zone, instanceName: targetVM });
|
||||
|
||||
// 2. Wake Worker
|
||||
const statusCheck = spawnSync(`gcloud compute instances describe ${targetVM} --project ${projectId} --zone ${zone} --format="get(status)"`, { shell: true });
|
||||
const status = statusCheck.stdout.toString().trim();
|
||||
|
||||
if (status !== 'RUNNING' && status !== 'PROVISIONING' && status !== 'STAGING') {
|
||||
console.log(`⚠️ Worker ${targetVM} is ${status}. Waking it up...`);
|
||||
spawnSync(`gcloud compute instances start ${targetVM} --project ${projectId} --zone ${zone}`, { shell: true, stdio: 'inherit' });
|
||||
}
|
||||
await provider.ensureReady();
|
||||
|
||||
const remotePolicyPath = `~/.gemini/policies/offload-policy.toml`;
|
||||
const persistentScripts = `~/.offload/scripts`;
|
||||
const sessionName = `offload-${prNumber}-${action}`;
|
||||
const remoteWorktreeDir = `~/dev/worktrees/${sessionName}`;
|
||||
const sshConfigPath = path.join(REPO_ROOT, '.gemini/offload_ssh_config');
|
||||
const sshBase = `ssh -F ${sshConfigPath}`;
|
||||
|
||||
// 3. Remote Context Setup (Parallel Worktree)
|
||||
console.log(`🚀 Provisioning persistent worktree for ${action} on #${prNumber}...`);
|
||||
|
||||
let setupCmd = '';
|
||||
if (action === 'implement') {
|
||||
// FIX: Always use explicit base (upstream/main) to prevent Branch Bleeding
|
||||
const branchName = `impl-${prNumber}`;
|
||||
setupCmd = `
|
||||
mkdir -p ~/dev/worktrees && \
|
||||
@@ -64,7 +61,6 @@ export async function runOrchestrator(args: string[], env: NodeJS.ProcessEnv = p
|
||||
git worktree add -f -b ${branchName} ${remoteWorktreeDir} upstream/main
|
||||
`;
|
||||
} else {
|
||||
// For PR-based actions, we fetch the PR head
|
||||
setupCmd = `
|
||||
mkdir -p ~/dev/worktrees && \
|
||||
cd ${remoteWorkDir} && \
|
||||
@@ -73,52 +69,19 @@ export async function runOrchestrator(args: string[], env: NodeJS.ProcessEnv = p
|
||||
`;
|
||||
}
|
||||
|
||||
// Wrap in docker exec if needed
|
||||
if (useContainer) {
|
||||
setupCmd = `docker exec maintainer-worker sh -c ${q(setupCmd)}`;
|
||||
}
|
||||
|
||||
spawnSync(`${sshBase} ${remoteHost} ${q(setupCmd)}`, { shell: true, stdio: 'inherit' });
|
||||
await provider.exec(setupCmd, { wrapContainer: 'maintainer-worker' });
|
||||
|
||||
// 4. Execution Logic (Persistent Workstation Mode)
|
||||
// We use docker exec if container mode is enabled, otherwise run on host.
|
||||
const remoteWorker = `tsx ${persistentScripts}/entrypoint.ts ${prNumber} remote-branch ${remotePolicyPath} ${action}`;
|
||||
|
||||
let tmuxCmd = `cd ${remoteWorktreeDir} && ${remoteWorker}; exec $SHELL`;
|
||||
if (useContainer) {
|
||||
// If in container mode, we jump into the shared 'maintainer-worker' container
|
||||
// We must use -i and -t for the interactive tmux session
|
||||
tmuxCmd = `docker exec -it -w /home/node/dev/worktrees/offload-${prNumber}-${action} maintainer-worker sh -c "${remoteWorker}; exec $SHELL"`;
|
||||
}
|
||||
// We launch a tmux session inside the container
|
||||
const tmuxCmd = `docker exec -it -w /home/node/dev/worktrees/${sessionName} maintainer-worker sh -c ${q(`${remoteWorker}; exec $SHELL`)}`;
|
||||
const tmuxAttach = `tmux attach-session -t ${sessionName} 2>/dev/null || tmux new-session -s ${sessionName} -n 'offload' ${q(tmuxCmd)}`;
|
||||
|
||||
const sshInternal = `tmux attach-session -t ${sessionName} 2>/dev/null || tmux new-session -s ${sessionName} -n 'offload' ${q(tmuxCmd)}`;
|
||||
|
||||
// High-performance primary SSH with IAP fallback
|
||||
const finalSSH = `${sshBase} -o ConnectTimeout=5 -t ${remoteHost} ${q(sshInternal)} || gcloud compute ssh ${targetVM} --project ${projectId} --zone ${zone} --tunnel-through-iap --command ${q(sshInternal)}`;
|
||||
// High-performance primary SSH with IAP fallback via Provider.exec
|
||||
// Note: We use provider.exec for consistency and robustness
|
||||
await provider.exec(tmuxAttach, { interactive: true });
|
||||
|
||||
// 5. Open in iTerm2
|
||||
const isWithinGemini = !!env.GEMINI_CLI || !!env.GEMINI_SESSION_ID || !!env.GCLI_SESSION_ID;
|
||||
if (isWithinGemini) {
|
||||
const tempCmdPath = path.join(process.env.TMPDIR || '/tmp', `offload-ssh-${prNumber}.sh`);
|
||||
fs.writeFileSync(tempCmdPath, `#!/bin/bash\n${finalSSH}\nrm "$0"`, { mode: 0o755 });
|
||||
|
||||
const appleScript = `
|
||||
on run argv
|
||||
tell application "iTerm"
|
||||
set newWindow to (create window with default profile)
|
||||
tell current session of newWindow
|
||||
write text (item 1 of argv) & return
|
||||
end tell
|
||||
activate
|
||||
end tell
|
||||
end run
|
||||
`;
|
||||
spawnSync('osascript', ['-', tempCmdPath], { input: appleScript });
|
||||
console.log(`✅ iTerm2 window opened on ${remoteHost} (Persistent Session).`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
spawnSync(finalSSH, { stdio: 'inherit', shell: true });
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -14,5 +14,5 @@ export async function runFixPlaybook(prNumber: string, targetDir: string, policy
|
||||
until the PR is fully passing and mergeable.`
|
||||
], { stdio: 'inherit' });
|
||||
|
||||
return result.status || 0;
|
||||
return result?.status ?? 1;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
/**
|
||||
* WorkerProvider interface defines the contract for different remote
|
||||
* execution environments (GCE, Workstations, etc.).
|
||||
*/
|
||||
export interface WorkerProvider {
|
||||
/**
|
||||
* Provisions the underlying infrastructure.
|
||||
*/
|
||||
provision(): Promise<number>;
|
||||
|
||||
/**
|
||||
* Ensures the worker is running and accessible.
|
||||
*/
|
||||
ensureReady(): Promise<number>;
|
||||
|
||||
/**
|
||||
* Performs the initial setup of the worker (SSH, scripts, auth).
|
||||
*/
|
||||
setup(options: SetupOptions): Promise<number>;
|
||||
|
||||
/**
|
||||
* Executes a command on the worker.
|
||||
*/
|
||||
exec(command: string, options?: ExecOptions): Promise<number>;
|
||||
|
||||
/**
|
||||
* Executes a command on the worker and returns the output.
|
||||
*/
|
||||
getExecOutput(command: string, options?: ExecOptions): Promise<{ status: number; stdout: string; stderr: string }>;
|
||||
|
||||
/**
|
||||
* Synchronizes local files to the worker.
|
||||
*/
|
||||
sync(localPath: string, remotePath: string, options?: SyncOptions): Promise<number>;
|
||||
|
||||
/**
|
||||
* Returns the status of the worker.
|
||||
*/
|
||||
getStatus(): Promise<WorkerStatus>;
|
||||
|
||||
/**
|
||||
* Stops the worker to save costs.
|
||||
*/
|
||||
stop(): Promise<number>;
|
||||
}
|
||||
|
||||
export interface SetupOptions {
|
||||
projectId: string;
|
||||
zone: string;
|
||||
dnsSuffix?: string;
|
||||
syncAuth?: boolean;
|
||||
}
|
||||
|
||||
export interface ExecOptions {
|
||||
interactive?: boolean;
|
||||
cwd?: string;
|
||||
wrapContainer?: string;
|
||||
}
|
||||
|
||||
export interface SyncOptions {
|
||||
delete?: boolean;
|
||||
exclude?: string[];
|
||||
}
|
||||
|
||||
export interface WorkerStatus {
|
||||
name: string;
|
||||
status: string;
|
||||
internalIp?: string;
|
||||
externalIp?: string;
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { spawnSync } from 'child_process';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import { WorkerProvider, SetupOptions, ExecOptions, SyncOptions, WorkerStatus } from './BaseProvider.ts';
|
||||
|
||||
export class GceCosProvider implements WorkerProvider {
|
||||
private projectId: string;
|
||||
private zone: string;
|
||||
private instanceName: string;
|
||||
private sshConfigPath: string;
|
||||
private knownHostsPath: string;
|
||||
private sshAlias = 'gcli-worker';
|
||||
|
||||
constructor(projectId: string, zone: string, instanceName: string, repoRoot: string) {
|
||||
this.projectId = projectId;
|
||||
this.zone = zone;
|
||||
this.instanceName = instanceName;
|
||||
this.sshConfigPath = path.join(repoRoot, '.gemini/offload_ssh_config');
|
||||
this.knownHostsPath = path.join(repoRoot, '.gemini/offload_known_hosts');
|
||||
}
|
||||
|
||||
async provision(): Promise<number> {
|
||||
const imageUri = 'us-docker.pkg.dev/gemini-code-dev/gemini-cli/maintainer:latest';
|
||||
console.log(`🚀 Provisioning GCE COS worker: ${this.instanceName}...`);
|
||||
|
||||
const startupScript = `#!/bin/bash
|
||||
docker pull ${imageUri}
|
||||
docker run -d --name maintainer-worker --restart always \\
|
||||
-v /home/node/dev:/home/node/dev:rw \\
|
||||
-v /home/node/.gemini:/home/node/.gemini:rw \\
|
||||
-v /home/node/.offload:/home/node/.offload:rw \\
|
||||
${imageUri} /bin/bash -c "while true; do sleep 1000; done"
|
||||
`;
|
||||
|
||||
const result = spawnSync('gcloud', [
|
||||
'compute', 'instances', 'create', this.instanceName,
|
||||
'--project', this.projectId,
|
||||
'--zone', this.zone,
|
||||
'--machine-type', 'n2-standard-8',
|
||||
'--image-family', 'cos-stable',
|
||||
'--image-project', 'cos-cloud',
|
||||
'--boot-disk-size', '200GB',
|
||||
'--boot-disk-type', 'pd-balanced',
|
||||
'--metadata', `startup-script=${startupScript},enable-oslogin=TRUE`,
|
||||
'--network-interface', 'network-tier=PREMIUM,no-address',
|
||||
'--scopes', 'https://www.googleapis.com/auth/cloud-platform'
|
||||
], { stdio: 'inherit' });
|
||||
|
||||
return result.status ?? 1;
|
||||
}
|
||||
|
||||
async ensureReady(): Promise<number> {
|
||||
const status = await this.getStatus();
|
||||
if (status.status !== 'RUNNING') {
|
||||
console.log(`⚠️ Worker ${this.instanceName} is ${status.status}. Waking it up...`);
|
||||
const res = spawnSync('gcloud', [
|
||||
'compute', 'instances', 'start', this.instanceName,
|
||||
'--project', this.projectId,
|
||||
'--zone', this.zone
|
||||
], { stdio: 'inherit' });
|
||||
if (res.status !== 0) return res.status ?? 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
async setup(options: SetupOptions): Promise<number> {
|
||||
const dnsSuffix = options.dnsSuffix || '.internal';
|
||||
|
||||
// Construct hostname. We attempt direct internal first.
|
||||
// We've removed 'nic0' by default as it was reported as inconsistent.
|
||||
const internalHostname = `${this.instanceName}.${this.zone}.c.${this.projectId}${dnsSuffix.startsWith('.') ? dnsSuffix : '.' + dnsSuffix}`;
|
||||
|
||||
const sshEntry = `
|
||||
Host ${this.sshAlias}
|
||||
HostName ${internalHostname}
|
||||
IdentityFile ~/.ssh/google_compute_engine
|
||||
User ${process.env.USER || 'node'}_google_com
|
||||
UserKnownHostsFile ${this.knownHostsPath}
|
||||
CheckHostIP no
|
||||
StrictHostKeyChecking no
|
||||
ConnectTimeout 5
|
||||
`;
|
||||
|
||||
fs.writeFileSync(this.sshConfigPath, sshEntry);
|
||||
console.log(` ✅ Created project SSH config: ${this.sshConfigPath}`);
|
||||
|
||||
console.log(' - Verifying connection and triggering SSO...');
|
||||
const directCheck = spawnSync('ssh', ['-F', this.sshConfigPath, this.sshAlias, 'echo 1'], { stdio: 'pipe', shell: true });
|
||||
|
||||
if (directCheck.status !== 0) {
|
||||
console.log(' ⚠️ Direct internal SSH failed. Attempting IAP tunnel fallback...');
|
||||
const iapCheck = spawnSync('gcloud', [
|
||||
'compute', 'ssh', this.instanceName,
|
||||
'--project', this.projectId,
|
||||
'--zone', this.zone,
|
||||
'--tunnel-through-iap',
|
||||
'--command', 'echo 1'
|
||||
], { stdio: 'inherit' });
|
||||
|
||||
if (iapCheck.status !== 0) {
|
||||
console.error('\n❌ All connection attempts failed. Please ensure you have "gcert" and IAP permissions.');
|
||||
return 1;
|
||||
}
|
||||
console.log(' ✅ IAP connection verified.');
|
||||
} else {
|
||||
console.log(' ✅ Direct internal connection verified.');
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
async exec(command: string, options: ExecOptions = {}): Promise<number> {
|
||||
const res = await this.getExecOutput(command, options);
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async getExecOutput(command: string, options: ExecOptions = {}): Promise<{ status: number; stdout: string; stderr: string }> {
|
||||
let finalCmd = command;
|
||||
if (options.wrapContainer) {
|
||||
finalCmd = `docker exec ${options.interactive ? '-it' : ''} ${options.cwd ? `-w ${options.cwd}` : ''} ${options.wrapContainer} sh -c ${this.quote(command)}`;
|
||||
}
|
||||
|
||||
const sshBase = ['ssh', '-F', this.sshConfigPath, options.interactive ? '-t' : '', this.sshAlias].filter(Boolean);
|
||||
const iapBase = [
|
||||
'gcloud', 'compute', 'ssh', this.instanceName,
|
||||
'--project', this.projectId,
|
||||
'--zone', this.zone,
|
||||
'--tunnel-through-iap',
|
||||
'--command'
|
||||
];
|
||||
|
||||
// Try direct first
|
||||
const directRes = spawnSync(sshBase[0], [...sshBase.slice(1), finalCmd], { stdio: options.interactive ? 'inherit' : 'pipe', shell: true });
|
||||
if (directRes.status === 0) {
|
||||
return {
|
||||
status: 0,
|
||||
stdout: directRes.stdout?.toString() || '',
|
||||
stderr: directRes.stderr?.toString() || ''
|
||||
};
|
||||
}
|
||||
|
||||
console.log('⚠️ Direct SSH failed, falling back to IAP...');
|
||||
const iapRes = spawnSync(iapBase[0], [...iapBase.slice(1), finalCmd], { stdio: options.interactive ? 'inherit' : 'pipe' });
|
||||
return {
|
||||
status: iapRes.status ?? 1,
|
||||
stdout: iapRes.stdout?.toString() || '',
|
||||
stderr: iapRes.stderr?.toString() || ''
|
||||
};
|
||||
}
|
||||
|
||||
async sync(localPath: string, remotePath: string, options: SyncOptions = {}): Promise<number> {
|
||||
const rsyncArgs = ['-avz', '--exclude=".gemini/settings.json"'];
|
||||
if (options.delete) rsyncArgs.push('--delete');
|
||||
if (options.exclude) {
|
||||
options.exclude.forEach(ex => rsyncArgs.push(`--exclude=${ex}`));
|
||||
}
|
||||
|
||||
const sshCmd = `ssh -F ${this.sshConfigPath} -o StrictHostKeyChecking=no -o UserKnownHostsFile=${this.knownHostsPath}`;
|
||||
|
||||
// Try direct rsync
|
||||
console.log(`📦 Syncing ${localPath} to ${this.sshAlias}:${remotePath}...`);
|
||||
const directRes = spawnSync('rsync', [...rsyncArgs, '-e', sshCmd, localPath, `${this.sshAlias}:${remotePath}`], { stdio: 'inherit', shell: true });
|
||||
|
||||
if (directRes.status === 0) return 0;
|
||||
|
||||
console.log('⚠️ Direct rsync failed, falling back to IAP-tunnelled rsync...');
|
||||
const iapSshCmd = `gcloud compute ssh --project ${this.projectId} --zone ${this.zone} --tunnel-through-iap --quiet`;
|
||||
const iapRes = spawnSync('rsync', [...rsyncArgs, '-e', iapSshCmd, localPath, `${this.instanceName}:${remotePath}`], { stdio: 'inherit', shell: true });
|
||||
|
||||
return iapRes.status ?? 1;
|
||||
}
|
||||
|
||||
async getStatus(): Promise<WorkerStatus> {
|
||||
const res = spawnSync('gcloud', [
|
||||
'compute', 'instances', 'describe', this.instanceName,
|
||||
'--project', this.projectId,
|
||||
'--zone', this.zone,
|
||||
'--format', 'json(name,status,networkInterfaces[0].networkIP)'
|
||||
], { stdio: 'pipe' });
|
||||
|
||||
if (res.status !== 0) {
|
||||
return { name: this.instanceName, status: 'UNKNOWN' };
|
||||
}
|
||||
|
||||
try {
|
||||
const data = JSON.parse(res.stdout.toString());
|
||||
return {
|
||||
name: data.name,
|
||||
status: data.status,
|
||||
internalIp: data.networkInterfaces?.[0]?.networkIP
|
||||
};
|
||||
} catch (e) {
|
||||
return { name: this.instanceName, status: 'ERROR' };
|
||||
}
|
||||
}
|
||||
|
||||
async stop(): Promise<number> {
|
||||
const res = spawnSync('gcloud', [
|
||||
'compute', 'instances', 'stop', this.instanceName,
|
||||
'--project', this.projectId,
|
||||
'--zone', this.zone
|
||||
], { stdio: 'inherit' });
|
||||
return res.status ?? 1;
|
||||
}
|
||||
|
||||
private quote(str: string) {
|
||||
return `'${str.replace(/'/g, "'\\''")}'`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { GceCosProvider } from './GceCosProvider.ts';
|
||||
import { WorkerProvider } from './BaseProvider.ts';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = path.resolve(__dirname, '../../../..');
|
||||
|
||||
export class ProviderFactory {
|
||||
static getProvider(config: { projectId: string; zone: string; instanceName: string }): WorkerProvider {
|
||||
// Currently we only have GceCosProvider, but this is where we'd branch
|
||||
return new GceCosProvider(config.projectId, config.zone, config.instanceName, REPO_ROOT);
|
||||
}
|
||||
}
|
||||
@@ -6,9 +6,9 @@
|
||||
import { spawnSync } from 'child_process';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import { fileURLToPath } from 'url';
|
||||
import readline from 'readline';
|
||||
import { ProviderFactory } from './providers/ProviderFactory.ts';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = path.resolve(__dirname, '../../../..');
|
||||
@@ -39,90 +39,55 @@ export async function runSetup(env: NodeJS.ProcessEnv = process.env) {
|
||||
const projectId = await prompt('GCP Project ID', 'gemini-cli-team-quota');
|
||||
const zone = await prompt('Compute Zone', 'us-west1-a');
|
||||
const targetVM = `gcli-offload-${env.USER || 'mattkorwel'}`;
|
||||
const useContainer = await confirm('Use Container-Native mode (Container-Optimized OS)?');
|
||||
|
||||
const provider = ProviderFactory.getProvider({ projectId, zone, instanceName: targetVM });
|
||||
|
||||
console.log(`🔍 Verifying access and finding worker ${targetVM}...`);
|
||||
const statusCheck = spawnSync(`gcloud compute instances describe ${targetVM} --project ${projectId} --zone ${zone} --format="json(status,networkInterfaces[0].networkIP)"`, { shell: true });
|
||||
const status = await provider.getStatus();
|
||||
|
||||
let instanceData: any;
|
||||
try {
|
||||
const output = statusCheck.stdout.toString().trim();
|
||||
if (!output) throw new Error('Empty output');
|
||||
instanceData = JSON.parse(output);
|
||||
} catch (e) {
|
||||
if (status.status === 'UNKNOWN' || status.status === 'ERROR') {
|
||||
console.error(`❌ Worker ${targetVM} not found or error fetching status. Run "npm run offload:fleet provision" first.`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const status = instanceData.status;
|
||||
|
||||
if (status !== 'RUNNING') {
|
||||
console.log(`⚠️ Worker is ${status}. Starting it for initialization...`);
|
||||
spawnSync(`gcloud compute instances start ${targetVM} --project ${projectId} --zone ${zone}`, { shell: true, stdio: 'inherit' });
|
||||
if (status.status !== 'RUNNING') {
|
||||
await provider.ensureReady();
|
||||
}
|
||||
|
||||
// 1. Configure Isolated SSH Alias (Direct Internal Hostname)
|
||||
console.log(`\n🚀 Configuring Isolated SSH Alias (Direct Internal Path)...`);
|
||||
const dnsSuffix = await prompt('Internal DNS Suffix', '.internal.gcpnode.com');
|
||||
// 1. Configure Isolated SSH Alias (Direct Internal Path with IAP Fallback)
|
||||
console.log(`\n🚀 Configuring Isolated SSH Alias...`);
|
||||
const dnsSuffix = await prompt('Internal DNS Suffix', '.internal');
|
||||
|
||||
// Construct the empirically verified high-performance hostname
|
||||
const internalHostname = `nic0.${targetVM}.${zone}.c.${projectId}${dnsSuffix.startsWith('.') ? dnsSuffix : '.' + dnsSuffix}`;
|
||||
const setupRes = await provider.setup({ projectId, zone, dnsSuffix });
|
||||
if (setupRes !== 0) return setupRes;
|
||||
|
||||
|
||||
const sshAlias = 'gcli-worker';
|
||||
const sshConfigPath = path.join(REPO_ROOT, '.gemini/offload_ssh_config');
|
||||
const knownHostsPath = path.join(REPO_ROOT, '.gemini/offload_known_hosts');
|
||||
|
||||
const sshEntry = `
|
||||
Host ${sshAlias}
|
||||
HostName ${internalHostname}
|
||||
IdentityFile ~/.ssh/google_compute_engine
|
||||
User ${env.USER || 'mattkorwel'}_google_com
|
||||
UserKnownHostsFile ${knownHostsPath}
|
||||
CheckHostIP no
|
||||
StrictHostKeyChecking no
|
||||
`;
|
||||
|
||||
fs.writeFileSync(sshConfigPath, sshEntry);
|
||||
console.log(` ✅ Created project SSH config: ${sshConfigPath}`);
|
||||
|
||||
// 1b. Security Fork Management (Temporarily Disabled)
|
||||
const upstreamRepo = 'google-gemini/gemini-cli';
|
||||
const userFork = upstreamRepo; // Fallback for now
|
||||
|
||||
// Resolve Paths
|
||||
const sshCmd = `ssh -F ${sshConfigPath} -o StrictHostKeyChecking=no -o UserKnownHostsFile=${knownHostsPath}`;
|
||||
const remoteHost = sshAlias;
|
||||
const remoteHome = '/home/node'; // Hardcoded for our maintainer container
|
||||
const remoteWorkDir = `/home/node/dev/main`; // Use absolute path
|
||||
const remoteWorkDir = `/home/node/dev/main`;
|
||||
const persistentScripts = `/home/node/.offload/scripts`;
|
||||
|
||||
console.log(`\n📦 Performing One-Time Synchronization...`);
|
||||
|
||||
// Trigger any SSH/SSO prompts before bulk sync
|
||||
console.log(' - Verifying connection and triggering SSO...');
|
||||
const connCheck = spawnSync(sshCmd, [remoteHost, 'echo 1'], { stdio: 'inherit', shell: true });
|
||||
if (connCheck.status !== 0) {
|
||||
console.error('\n❌ SSH connection failed. Please ensure you have run "gcert" recently.');
|
||||
return 1;
|
||||
}
|
||||
// Ensure host directories exist (using provider.exec to handle IAP fallback)
|
||||
await provider.exec(`mkdir -p /home/node/dev/main /home/node/.gemini/policies /home/node/.offload/scripts`);
|
||||
|
||||
// Ensure host directories exist (on the VM Host)
|
||||
spawnSync(sshCmd, [remoteHost, `mkdir -p /home/node/dev/main /home/node/.gemini/policies /home/node/.offload/scripts`], { shell: true });
|
||||
|
||||
const rsyncBase = `rsync -avz -e "${sshCmd}" --exclude=".gemini/settings.json"`;
|
||||
|
||||
// 2. Sync Scripts & Policies
|
||||
console.log(' - Pushing offload logic to persistent worker directory...');
|
||||
spawnSync(`${rsyncBase} --delete .gemini/skills/offload/scripts/ ${remoteHost}:${persistentScripts}/`, { shell: true });
|
||||
spawnSync(`${rsyncBase} .gemini/skills/offload/policy.toml ${remoteHost}:/home/node/.gemini/policies/offload-policy.toml`, { shell: true });
|
||||
await provider.sync('.gemini/skills/offload/scripts/', `${persistentScripts}/`, { delete: true });
|
||||
await provider.sync('.gemini/skills/offload/policy.toml', `/home/node/.gemini/policies/offload-policy.toml`);
|
||||
|
||||
// 3. Sync Auth (Gemini)
|
||||
if (await confirm('Sync Gemini accounts credentials?')) {
|
||||
const homeDir = env.HOME || '';
|
||||
const lp = path.join(homeDir, '.gemini/google_accounts.json');
|
||||
if (fs.existsSync(lp)) {
|
||||
spawnSync(`${rsyncBase} ${lp} ${remoteHost}:/home/node/.gemini/google_accounts.json`, { shell: true });
|
||||
await provider.sync(lp, `/home/node/.gemini/google_accounts.json`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -146,22 +111,18 @@ Host ${sshAlias}
|
||||
|
||||
const scopedToken = await prompt('\nPaste Scoped Token', '');
|
||||
if (scopedToken) {
|
||||
spawnSync(sshCmd, [remoteHost, `mkdir -p /home/node/.offload && echo ${scopedToken} > /home/node/.offload/.gh_token && chmod 600 /home/node/.offload/.gh_token`], { shell: true });
|
||||
await provider.exec(`mkdir -p /home/node/.offload && echo ${scopedToken} > /home/node/.offload/.gh_token && chmod 600 /home/node/.offload/.gh_token`);
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Tooling & Clone
|
||||
if (await confirm('Initialize tools and clone repository?')) {
|
||||
if (!useContainer) {
|
||||
spawnSync(sshCmd, [remoteHost, `sudo npm install -g tsx vitest`], { shell: true, stdio: 'inherit' });
|
||||
}
|
||||
|
||||
console.log(`🚀 Cloning fork ${userFork} on worker...`);
|
||||
const repoUrl = `https://github.com/${userFork}.git`;
|
||||
|
||||
// Wipe existing dir for a clean clone and use absolute paths
|
||||
const cloneCmd = `rm -rf ${remoteWorkDir} && git clone --filter=blob:none ${repoUrl} ${remoteWorkDir} && cd ${remoteWorkDir} && git remote add upstream https://github.com/${upstreamRepo}.git && git fetch upstream`;
|
||||
spawnSync(sshCmd, [remoteHost, cloneCmd], { shell: true, stdio: 'inherit' });
|
||||
await provider.exec(cloneCmd);
|
||||
}
|
||||
|
||||
// Save Settings
|
||||
@@ -172,9 +133,10 @@ Host ${sshAlias}
|
||||
}
|
||||
settings.maintainer = settings.maintainer || {};
|
||||
settings.maintainer.deepReview = {
|
||||
projectId, zone, remoteHost,
|
||||
projectId, zone,
|
||||
remoteHost: 'gcli-worker',
|
||||
remoteWorkDir, userFork, upstreamRepo,
|
||||
useContainer,
|
||||
useContainer: true,
|
||||
terminalType: 'iterm2'
|
||||
};
|
||||
fs.writeFileSync(settingsPath, JSON.stringify(settings, null, 2));
|
||||
@@ -186,3 +148,4 @@ Host ${sshAlias}
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
runSetup().catch(console.error);
|
||||
}
|
||||
|
||||
|
||||
@@ -4,10 +4,12 @@ import fs from 'fs';
|
||||
import readline from 'readline';
|
||||
import { runOrchestrator } from '../scripts/orchestrator.ts';
|
||||
import { runWorker } from '../scripts/worker.ts';
|
||||
import { ProviderFactory } from '../scripts/providers/ProviderFactory.ts';
|
||||
|
||||
vi.mock('child_process');
|
||||
vi.mock('fs');
|
||||
vi.mock('readline');
|
||||
vi.mock('../scripts/providers/ProviderFactory.ts');
|
||||
|
||||
describe('Offload Tooling Matrix', () => {
|
||||
const mockSettings = {
|
||||
@@ -15,37 +17,31 @@ describe('Offload Tooling Matrix', () => {
|
||||
deepReview: {
|
||||
projectId: 'test-project',
|
||||
zone: 'us-west1-a',
|
||||
terminalType: 'none',
|
||||
syncAuth: false,
|
||||
geminiSetup: 'isolated',
|
||||
ghSetup: 'isolated'
|
||||
remoteWorkDir: '/home/node/dev/main'
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const mockProvider = {
|
||||
provision: vi.fn().mockResolvedValue(0),
|
||||
ensureReady: vi.fn().mockResolvedValue(0),
|
||||
setup: vi.fn().mockResolvedValue(0),
|
||||
exec: vi.fn().mockResolvedValue(0),
|
||||
getExecOutput: vi.fn().mockResolvedValue({ status: 0, stdout: '', stderr: '' }),
|
||||
sync: vi.fn().mockResolvedValue(0),
|
||||
getStatus: vi.fn().mockResolvedValue({ name: 'test-instance', status: 'RUNNING' }),
|
||||
stop: vi.fn().mockResolvedValue(0)
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.clearAllMocks();
|
||||
vi.mocked(fs.existsSync).mockReturnValue(true);
|
||||
vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify(mockSettings));
|
||||
vi.mocked(fs.mkdirSync).mockReturnValue(undefined as any);
|
||||
vi.mocked(fs.writeFileSync).mockReturnValue(undefined as any);
|
||||
vi.mocked(fs.createWriteStream).mockReturnValue({ pipe: vi.fn() } as any);
|
||||
vi.spyOn(process, 'chdir').mockImplementation(() => {});
|
||||
vi.mocked(ProviderFactory.getProvider).mockReturnValue(mockProvider as any);
|
||||
|
||||
vi.mocked(spawnSync).mockImplementation((cmd: any, args: any) => {
|
||||
const callStr = JSON.stringify({ cmd, args });
|
||||
|
||||
// 1. Mock GCloud Instance List
|
||||
if (callStr.includes('gcloud') && callStr.includes('instances') && callStr.includes('list')) {
|
||||
return { status: 0, stdout: Buffer.from(JSON.stringify([{ name: 'gcli-offload-test-worker' }])), stderr: Buffer.from('') } as any;
|
||||
}
|
||||
|
||||
// 2. Mock GH Metadata Fetching (local or remote)
|
||||
if (callStr.includes('gh') && callStr.includes('view')) {
|
||||
return { status: 0, stdout: Buffer.from('test-meta\n'), stderr: Buffer.from('') } as any;
|
||||
}
|
||||
|
||||
return { status: 0, stdout: Buffer.from(''), stderr: Buffer.from('') } as any;
|
||||
vi.mocked(spawnSync).mockImplementation((cmd: any) => {
|
||||
if (cmd === 'gh') return { status: 0, stdout: Buffer.from('test-branch\n') } as any;
|
||||
return { status: 0, stdout: Buffer.from('') } as any;
|
||||
});
|
||||
|
||||
vi.mocked(spawn).mockImplementation(() => {
|
||||
@@ -56,24 +52,16 @@ describe('Offload Tooling Matrix', () => {
|
||||
pid: 1234
|
||||
} as any;
|
||||
});
|
||||
|
||||
vi.spyOn(process, 'chdir').mockImplementation(() => {});
|
||||
});
|
||||
|
||||
describe('Implement Playbook', () => {
|
||||
it('should create a branch and run research/implementation', async () => {
|
||||
await runOrchestrator(['456', 'implement'], {});
|
||||
|
||||
const spawnCalls = vi.mocked(spawnSync).mock.calls;
|
||||
const ghCall = spawnCalls.find(call => {
|
||||
const s = JSON.stringify(call);
|
||||
return s.includes('gh') && s.includes('issue') && s.includes('view') && s.includes('456');
|
||||
});
|
||||
expect(ghCall).toBeDefined();
|
||||
|
||||
const sshCall = spawnCalls.find(call => {
|
||||
const s = JSON.stringify(call);
|
||||
return s.includes('gcloud') && s.includes('ssh') && s.includes('offload-456-implement');
|
||||
});
|
||||
expect(sshCall).toBeDefined();
|
||||
expect(mockProvider.exec).toHaveBeenCalledWith(expect.stringContaining('git worktree add'), expect.any(Object));
|
||||
expect(mockProvider.exec).toHaveBeenCalledWith(expect.stringContaining('tmux new-session'), expect.any(Object));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -81,6 +69,7 @@ describe('Offload Tooling Matrix', () => {
|
||||
it('should launch the agentic fix-pr skill', async () => {
|
||||
vi.mocked(fs.existsSync).mockReturnValue(true);
|
||||
await runWorker(['123', 'test-branch', '/path/policy', 'fix']);
|
||||
|
||||
const spawnSyncCalls = vi.mocked(spawnSync).mock.calls;
|
||||
const fixCall = spawnSyncCalls.find(call =>
|
||||
JSON.stringify(call).includes("activate the 'fix-pr' skill")
|
||||
|
||||
@@ -1,85 +1,61 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { spawnSync, spawn } from 'child_process';
|
||||
import { spawnSync } from 'child_process';
|
||||
import fs from 'fs';
|
||||
import readline from 'readline';
|
||||
import { runOrchestrator } from '../scripts/orchestrator.ts';
|
||||
import { runSetup } from '../scripts/setup.ts';
|
||||
import { runWorker } from '../scripts/worker.ts';
|
||||
import { ProviderFactory } from '../scripts/providers/ProviderFactory.ts';
|
||||
|
||||
vi.mock('child_process');
|
||||
vi.mock('fs');
|
||||
vi.mock('readline');
|
||||
vi.mock('../scripts/providers/ProviderFactory.ts');
|
||||
|
||||
describe('Offload Orchestration (GCE)', () => {
|
||||
describe('Offload Orchestration (Refactored)', () => {
|
||||
const mockSettings = {
|
||||
maintainer: {
|
||||
deepReview: {
|
||||
projectId: 'test-project',
|
||||
zone: 'us-west1-a',
|
||||
terminalType: 'none',
|
||||
syncAuth: false,
|
||||
geminiSetup: 'isolated',
|
||||
ghSetup: 'isolated'
|
||||
remoteWorkDir: '/home/node/dev/main'
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const mockProvider = {
|
||||
provision: vi.fn().mockResolvedValue(0),
|
||||
ensureReady: vi.fn().mockResolvedValue(0),
|
||||
setup: vi.fn().mockResolvedValue(0),
|
||||
exec: vi.fn().mockResolvedValue(0),
|
||||
getExecOutput: vi.fn().mockResolvedValue({ status: 0, stdout: '', stderr: '' }),
|
||||
sync: vi.fn().mockResolvedValue(0),
|
||||
getStatus: vi.fn().mockResolvedValue({ name: 'test-instance', status: 'RUNNING' }),
|
||||
stop: vi.fn().mockResolvedValue(0)
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.clearAllMocks();
|
||||
vi.mocked(fs.existsSync).mockReturnValue(true);
|
||||
vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify(mockSettings));
|
||||
vi.mocked(fs.mkdirSync).mockReturnValue(undefined as any);
|
||||
vi.mocked(fs.writeFileSync).mockReturnValue(undefined as any);
|
||||
vi.mocked(fs.createWriteStream).mockReturnValue({ pipe: vi.fn() } as any);
|
||||
|
||||
// Explicitly set the mock return value for each test
|
||||
vi.mocked(ProviderFactory.getProvider).mockReturnValue(mockProvider as any);
|
||||
|
||||
vi.mocked(spawnSync).mockImplementation((cmd: any) => {
|
||||
if (cmd === 'gh') return { status: 0, stdout: Buffer.from('test-branch\n') } as any;
|
||||
return { status: 0, stdout: Buffer.from('') } as any;
|
||||
});
|
||||
|
||||
vi.spyOn(process, 'chdir').mockImplementation(() => {});
|
||||
vi.spyOn(process, 'cwd').mockReturnValue('/test-cwd');
|
||||
|
||||
// Default mock for gcloud instance info and describe
|
||||
vi.mocked(spawnSync).mockImplementation((cmd: any, args: any) => {
|
||||
const callInfo = JSON.stringify({ cmd, args });
|
||||
if (callInfo.includes('compute') && callInfo.includes('describe')) {
|
||||
return { status: 0, stdout: Buffer.from('RUNNING\n'), stderr: Buffer.from('') } as any;
|
||||
}
|
||||
if (callInfo.includes('gcloud') && callInfo.includes('ssh') && callInfo.includes('pwd')) {
|
||||
return { status: 0, stdout: Buffer.from('/home/testuser\n'), stderr: Buffer.from('') } as any;
|
||||
}
|
||||
if (callInfo.includes('gh') && callInfo.includes('view')) {
|
||||
return { status: 0, stdout: Buffer.from('test-meta\n'), stderr: Buffer.from('') } as any;
|
||||
}
|
||||
return { status: 0, stdout: Buffer.from(''), stderr: Buffer.from('') } as any;
|
||||
});
|
||||
|
||||
vi.mocked(spawn).mockImplementation(() => {
|
||||
return {
|
||||
stdout: { pipe: vi.fn(), on: vi.fn() },
|
||||
stderr: { pipe: vi.fn(), on: vi.fn() },
|
||||
on: vi.fn((event, cb) => { if (event === 'close') cb(0); }),
|
||||
pid: 1234
|
||||
} as any;
|
||||
});
|
||||
});
|
||||
|
||||
describe('orchestrator.ts', () => {
|
||||
it('should connect to the deterministic worker and use gcloud compute ssh', async () => {
|
||||
it('should wake the worker and execute remote commands', async () => {
|
||||
await runOrchestrator(['123'], { USER: 'testuser' });
|
||||
|
||||
const spawnCalls = vi.mocked(spawnSync).mock.calls;
|
||||
const sshCall = spawnCalls.find(call =>
|
||||
JSON.stringify(call).includes('gcloud') && JSON.stringify(call).includes('ssh')
|
||||
);
|
||||
|
||||
expect(sshCall).toBeDefined();
|
||||
// Match the new deterministic name: gcli-offload-<USER>
|
||||
expect(JSON.stringify(sshCall)).toContain('gcli-offload-testuser');
|
||||
expect(JSON.stringify(sshCall)).toContain('test-project');
|
||||
});
|
||||
|
||||
it('should construct the correct tmux session name', async () => {
|
||||
await runOrchestrator(['123'], {});
|
||||
const spawnCalls = vi.mocked(spawnSync).mock.calls;
|
||||
const sshCall = spawnCalls.find(call => JSON.stringify(call).includes('tmux new-session'));
|
||||
expect(JSON.stringify(sshCall)).toContain('offload-123-review');
|
||||
expect(mockProvider.ensureReady).toHaveBeenCalled();
|
||||
expect(mockProvider.exec).toHaveBeenCalledWith(expect.stringContaining('git worktree add'), expect.any(Object));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -93,31 +69,21 @@ describe('Offload Orchestration (GCE)', () => {
|
||||
vi.mocked(readline.createInterface).mockReturnValue(mockInterface as any);
|
||||
});
|
||||
|
||||
it('should verify project access during setup', async () => {
|
||||
vi.mocked(spawnSync).mockImplementation((cmd: any) => {
|
||||
if (cmd === 'gcloud') return { status: 0 } as any;
|
||||
return { status: 0, stdout: Buffer.from('') } as any;
|
||||
});
|
||||
|
||||
it('should use the provider to configure SSH and sync scripts', async () => {
|
||||
mockInterface.question
|
||||
.mockImplementationOnce((q, cb) => cb('test-project'))
|
||||
.mockImplementationOnce((q, cb) => cb('us-west1-a'))
|
||||
.mockImplementationOnce((q, cb) => cb('n2-standard-8'))
|
||||
.mockImplementationOnce((q, cb) => cb('y')) // syncAuth
|
||||
.mockImplementationOnce((q, cb) => cb('none'));
|
||||
.mockImplementationOnce((q, cb) => cb('.internal')) // dnsSuffix
|
||||
.mockImplementationOnce((q, cb) => cb('n')) // sync auth
|
||||
.mockImplementationOnce((q, cb) => cb('n')) // scoped token
|
||||
.mockImplementationOnce((q, cb) => cb('n')); // clone
|
||||
|
||||
await runSetup({ HOME: '/test-home' });
|
||||
// Ensure mockProvider is returned
|
||||
vi.mocked(ProviderFactory.getProvider).mockReturnValue(mockProvider as any);
|
||||
|
||||
expect(vi.mocked(spawnSync)).toHaveBeenCalledWith('gcloud', expect.arrayContaining(['projects', 'describe', 'test-project']), expect.any(Object));
|
||||
});
|
||||
});
|
||||
await runSetup({ USER: 'testuser' });
|
||||
|
||||
describe('worker.ts (playbooks)', () => {
|
||||
it('should launch the review playbook', async () => {
|
||||
vi.mocked(fs.existsSync).mockReturnValue(true);
|
||||
await runWorker(['123', 'test-branch', '/test-policy.toml', 'review']);
|
||||
const spawnCalls = vi.mocked(spawn).mock.calls;
|
||||
expect(spawnCalls.some(c => JSON.stringify(c).includes("activate the 'review-pr' skill"))).toBe(true);
|
||||
expect(mockProvider.setup).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { spawnSync, spawn } from 'child_process';
|
||||
import { spawnSync } from 'child_process';
|
||||
import fs from 'fs';
|
||||
import { runFixPlaybook } from '../../scripts/playbooks/fix.ts';
|
||||
|
||||
@@ -9,26 +9,15 @@ vi.mock('fs');
|
||||
describe('Fix Playbook', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.mocked(fs.mkdirSync).mockReturnValue(undefined as any);
|
||||
vi.mocked(fs.writeFileSync).mockReturnValue(undefined as any);
|
||||
vi.mocked(fs.createWriteStream).mockReturnValue({ pipe: vi.fn() } as any);
|
||||
|
||||
vi.mocked(spawn).mockImplementation(() => {
|
||||
return {
|
||||
stdout: { pipe: vi.fn(), on: vi.fn() },
|
||||
stderr: { pipe: vi.fn(), on: vi.fn() },
|
||||
on: vi.fn((event, cb) => { if (event === 'close') cb(0); })
|
||||
} as any;
|
||||
});
|
||||
vi.mocked(spawnSync).mockReturnValue({ status: 0 } as any);
|
||||
});
|
||||
|
||||
it('should register and run initial build, failure analysis, and fixer', async () => {
|
||||
runFixPlaybook('123', '/tmp/target', '/path/policy', '/path/gemini');
|
||||
it('should launch the agentic fix-pr skill via spawnSync', async () => {
|
||||
const status = await runFixPlaybook('123', '/tmp/target', '/path/policy', '/path/gemini');
|
||||
|
||||
const spawnCalls = vi.mocked(spawn).mock.calls;
|
||||
expect(status).toBe(0);
|
||||
const spawnCalls = vi.mocked(spawnSync).mock.calls;
|
||||
|
||||
expect(spawnCalls.some(c => c[0].includes('npm ci'))).toBe(true);
|
||||
expect(spawnCalls.some(c => c[0].includes('gh run view --log-failed'))).toBe(true);
|
||||
expect(spawnCalls.some(c => c[0].includes('Gemini Fixer'))).toBe(false); // Should wait for build
|
||||
expect(spawnCalls.some(c => JSON.stringify(c).includes("activate the 'fix-pr' skill"))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { spawnSync, spawn } from 'child_process';
|
||||
import { spawn } from 'child_process';
|
||||
import fs from 'fs';
|
||||
import { runReviewPlaybook } from '../../scripts/playbooks/review.ts';
|
||||
|
||||
@@ -22,16 +22,15 @@ describe('Review Playbook', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('should register and run build, ci, analysis, and verification', async () => {
|
||||
const promise = runReviewPlaybook('123', '/tmp/target', '/path/policy', '/path/gemini');
|
||||
it('should register and run build, ci, and review tasks', async () => {
|
||||
// We don't await because TaskRunner uses setInterval and we'd need to mock timers
|
||||
// but we can check if spawn was called with the right commands.
|
||||
runReviewPlaybook('123', '/tmp/target', '/path/policy', '/path/gemini');
|
||||
|
||||
// The worker uses setInterval(1500) to check for completion, so we need to wait
|
||||
// or mock the timer. For simplicity in this POC, we'll just verify spawn calls.
|
||||
const spawnCalls = vi.mocked(spawn).mock.calls;
|
||||
|
||||
// These should start immediately (no deps)
|
||||
expect(spawnCalls.some(c => c[0].includes('npm ci'))).toBe(true);
|
||||
expect(spawnCalls.some(c => c[0].includes('gh pr checks'))).toBe(true);
|
||||
expect(spawnCalls.some(c => c[0].includes('/review-frontend'))).toBe(true);
|
||||
expect(spawnCalls.some(c => c[0].includes("activate the 'review-pr' skill"))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { spawnSync } from 'child_process';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { GceCosProvider } from '../scripts/providers/GceCosProvider.ts';
|
||||
|
||||
vi.mock('child_process');
|
||||
vi.mock('fs');
|
||||
|
||||
describe('GceCosProvider', () => {
|
||||
const mockConfig = {
|
||||
projectId: 'test-project',
|
||||
zone: 'us-west1-a',
|
||||
instanceName: 'test-instance',
|
||||
repoRoot: '/test-root'
|
||||
};
|
||||
|
||||
let provider: GceCosProvider;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
provider = new GceCosProvider(mockConfig.projectId, mockConfig.zone, mockConfig.instanceName, mockConfig.repoRoot);
|
||||
|
||||
vi.mocked(spawnSync).mockReturnValue({ status: 0, stdout: Buffer.from(''), stderr: Buffer.from('') } as any);
|
||||
});
|
||||
|
||||
it('should provision an instance with COS image and startup script', async () => {
|
||||
await provider.provision();
|
||||
|
||||
const calls = vi.mocked(spawnSync).mock.calls;
|
||||
const createCall = calls.find(c => c[1].includes('create'));
|
||||
|
||||
expect(createCall).toBeDefined();
|
||||
expect(createCall![1]).toContain('cos-stable');
|
||||
expect(createCall![1]).toContain('test-instance');
|
||||
});
|
||||
|
||||
it('should attempt direct SSH and fallback to IAP on failure', async () => {
|
||||
// Fail direct SSH
|
||||
vi.mocked(spawnSync)
|
||||
.mockReturnValueOnce({ status: 1, stdout: Buffer.from(''), stderr: Buffer.from('fail') } as any) // direct
|
||||
.mockReturnValueOnce({ status: 0, stdout: Buffer.from('ok'), stderr: Buffer.from('') } as any); // IAP
|
||||
|
||||
const result = await provider.exec('echo 1');
|
||||
|
||||
expect(result).toBe(0);
|
||||
const calls = vi.mocked(spawnSync).mock.calls;
|
||||
expect(calls[0][0]).toBe('ssh');
|
||||
expect(calls[1][1]).toContain('--tunnel-through-iap');
|
||||
});
|
||||
|
||||
it('should sync files with IAP fallback', async () => {
|
||||
// Fail direct rsync
|
||||
vi.mocked(spawnSync)
|
||||
.mockReturnValueOnce({ status: 1 } as any) // direct
|
||||
.mockReturnValueOnce({ status: 0 } as any); // IAP
|
||||
|
||||
await provider.sync('./local', '/remote');
|
||||
|
||||
const calls = vi.mocked(spawnSync).mock.calls;
|
||||
expect(calls[0][0]).toBe('rsync');
|
||||
expect(calls[1][1]).toContain('gcloud compute ssh --project test-project --zone us-west1-a --tunnel-through-iap --quiet');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user