Roll back sandbox create destroy workflow to Monday

This commit is contained in:
mmckeen-nv
2026-04-30 10:08:44 -07:00
parent ae7a7c3a26
commit fda3cbb056
7 changed files with 181 additions and 536 deletions
+1 -93
View File
@@ -2,7 +2,6 @@ import { NextResponse } from "next/server"
import { execFile, spawn } from "node:child_process"
import { promisify } from "node:util"
import { inspectSandbox } from "@/app/lib/openshellHost"
import { repairOpenClawWorkspacePermissions, stabilizeOpenClawGatewayConfig } from "@/app/lib/sandboxPrivilegedFiles"
import {
commandExists,
HOST_PATH,
@@ -58,60 +57,6 @@ type NemoClawCreateCommand = {
mode: "cli-onboard" | "legacy-setup"
}
type CreateInferenceMode = "auto" | "vllm" | "nim"
type CreateInferenceSettings = {
mode: CreateInferenceMode
model: string
hasApiKey: boolean
envSummary: string[]
}
function parseCreateInferenceSettings(body: any): CreateInferenceSettings {
const raw = body?.createInference && typeof body.createInference === "object" ? body.createInference : {}
const requestedMode = typeof raw.mode === "string" ? raw.mode : "auto"
const mode: CreateInferenceMode = requestedMode === "vllm" || requestedMode === "nim" ? requestedMode : "auto"
const model = typeof raw.model === "string" ? raw.model.trim() : ""
const apiKey = typeof raw.apiKey === "string" ? raw.apiKey.trim() : ""
return {
mode,
model,
hasApiKey: Boolean(apiKey),
envSummary: [],
}
}
function applyCreateInferenceEnv(env: NodeJS.ProcessEnv, settings: CreateInferenceSettings, body: any) {
if (settings.mode === "auto") return settings
env.NEMOCLAW_EXPERIMENTAL = "1"
settings.envSummary.push("NEMOCLAW_EXPERIMENTAL=1")
if (settings.mode === "vllm") {
env.NEMOCLAW_PROVIDER = "vllm"
settings.envSummary.push("NEMOCLAW_PROVIDER=vllm")
}
if (settings.mode === "nim") {
env.NEMOCLAW_PROVIDER = "nim-local"
settings.envSummary.push("NEMOCLAW_PROVIDER=nim-local")
const apiKey = typeof body?.createInference?.apiKey === "string" ? body.createInference.apiKey.trim() : ""
if (apiKey) {
env.NVIDIA_API_KEY = apiKey
env.NEMOCLAW_PROVIDER_KEY = apiKey
settings.envSummary.push("NVIDIA_API_KEY=<provided>", "NEMOCLAW_PROVIDER_KEY=<provided>")
}
}
if (settings.model) {
env.NEMOCLAW_MODEL = settings.model
settings.envSummary.push(`NEMOCLAW_MODEL=${settings.model}`)
}
return settings
}
function buildNemoClawCreateCommand(): NemoClawCreateCommand {
if (NEMOCLAW_BIN && commandExists(NEMOCLAW_BIN)) {
const args = ["onboard", "--non-interactive", "--recreate-sandbox", "--yes-i-accept-third-party-software"]
@@ -386,8 +331,7 @@ export async function POST(request: Request) {
const sandboxName = validateSandboxName(typeof body?.sandboxName === "string" ? body.sandboxName.trim() : "")
const policy = body?.policy && typeof body.policy === "object" ? body.policy : null
const enableTailscale = Boolean(body?.enableTailscale)
const createInference = parseCreateInferenceSettings(body)
console.log(`[sandbox/create] request:parsed sandbox=${sandboxName} blueprint=${blueprint} enableTailscale=${enableTailscale} inferenceMode=${createInference.mode} elapsedMs=${elapsedMs(requestStartedAt)}`)
console.log(`[sandbox/create] request:parsed sandbox=${sandboxName} blueprint=${blueprint} enableTailscale=${enableTailscale} elapsedMs=${elapsedMs(requestStartedAt)}`)
if (!blueprint) {
return NextResponse.json({ ok: false, error: "blueprint is required" }, { status: 400 })
@@ -409,8 +353,6 @@ export async function POST(request: Request) {
env.NVIDIA_API_KEY = env.NVIDIA_API_KEY || "optional-local-mode"
}
applyCreateInferenceEnv(env, createInference, body)
const result = await runCommand(
createCommand.file,
createCommand.mode === "legacy-setup" ? [...createCommand.args, sandboxName] : createCommand.args,
@@ -427,7 +369,6 @@ export async function POST(request: Request) {
blueprint,
sandboxName,
enableTailscale,
createInference,
}, { status: 500 })
}
@@ -438,16 +379,6 @@ export async function POST(request: Request) {
error: "Sandbox readiness polling produced no verification result.",
}
const created = readiness.verified
const gatewayConfigRepair = created ? await stabilizeOpenClawGatewayConfig(sandboxName).catch((error) => ({
sandboxName,
path: "/sandbox/.openclaw/openclaw.json",
error: error instanceof Error ? error.message : "Failed to stabilize OpenClaw gateway config",
})) : null
const workspaceRepair = created ? await repairOpenClawWorkspacePermissions(sandboxName).catch((error) => ({
sandboxName,
path: "/sandbox/.openclaw/workspace",
error: error instanceof Error ? error.message : "Failed to repair OpenClaw workspace permissions",
})) : null
const deviceApproval = created ? await approveOpenClawDeviceRequests(sandboxName) : null
console.log(
`[sandbox/create] request:complete sandbox=${sandboxName} created=${created} readinessAttempts=${readiness.attempts} deviceApproval=${deviceApproval?.approved ?? false} elapsedMs=${elapsedMs(requestStartedAt)}`,
@@ -462,15 +393,12 @@ export async function POST(request: Request) {
verification,
mode: "nemoclaw-blueprint",
enableTailscale,
createInference,
createCommand,
hostPath: HOST_PATH,
readiness: {
attempts: readiness.attempts,
elapsedMs: readiness.elapsedMs,
},
workspaceRepair,
gatewayConfigRepair,
deviceApproval,
stdout: result.stdout,
stderr: result.stderr,
@@ -479,10 +407,6 @@ export async function POST(request: Request) {
enableTailscale
? "NemoClaw blueprint workflow completed with Tailscale-enabled prerequisites. Existing healthy OpenShell gateways are reused before any new gateway start is attempted."
: "NemoClaw blueprint workflow completed in local/default mode. Existing healthy OpenShell gateways are reused before any new gateway start is attempted.",
workspaceRepair && "note" in workspaceRepair ? workspaceRepair.note : false,
workspaceRepair && "error" in workspaceRepair ? `OpenClaw workspace permission repair failed: ${workspaceRepair.error}` : false,
gatewayConfigRepair && "note" in gatewayConfigRepair ? gatewayConfigRepair.note : false,
gatewayConfigRepair && "error" in gatewayConfigRepair ? `OpenClaw gateway config repair failed: ${gatewayConfigRepair.error}` : false,
deviceApproval?.note,
)
: "Blueprint command reported success, but the sandbox never reached a ready verification state afterward.",
@@ -517,16 +441,6 @@ export async function POST(request: Request) {
error: "Sandbox readiness polling produced no verification result.",
}
const created = readiness.verified
const gatewayConfigRepair = created ? await stabilizeOpenClawGatewayConfig(sandboxName).catch((error) => ({
sandboxName,
path: "/sandbox/.openclaw/openclaw.json",
error: error instanceof Error ? error.message : "Failed to stabilize OpenClaw gateway config",
})) : null
const workspaceRepair = created ? await repairOpenClawWorkspacePermissions(sandboxName).catch((error) => ({
sandboxName,
path: "/sandbox/.openclaw/workspace",
error: error instanceof Error ? error.message : "Failed to repair OpenClaw workspace permissions",
})) : null
const deviceApproval = created ? await approveOpenClawDeviceRequests(sandboxName) : null
const policyPrepared = Boolean(policy)
console.log(
@@ -553,8 +467,6 @@ export async function POST(request: Request) {
attempts: readiness.attempts,
elapsedMs: readiness.elapsedMs,
},
workspaceRepair,
gatewayConfigRepair,
deviceApproval,
policyPrepared,
note: created
@@ -564,10 +476,6 @@ export async function POST(request: Request) {
: (policyPrepared
? "Custom sandbox created. Policy draft is prepared, but applying it live should be a follow-up action."
: "Custom sandbox created."),
workspaceRepair && "note" in workspaceRepair ? workspaceRepair.note : false,
workspaceRepair && "error" in workspaceRepair ? `OpenClaw workspace permission repair failed: ${workspaceRepair.error}` : false,
gatewayConfigRepair && "note" in gatewayConfigRepair ? gatewayConfigRepair.note : false,
gatewayConfigRepair && "error" in gatewayConfigRepair ? `OpenClaw gateway config repair failed: ${gatewayConfigRepair.error}` : false,
deviceApproval?.note,
)
: "Create command started, but the sandbox never reached a ready verification state afterward.",
+43 -46
View File
@@ -1,10 +1,8 @@
import { NextResponse } from "next/server"
import { execFile } from "node:child_process"
import { mkdir, readFile, writeFile } from "node:fs/promises"
import path from "node:path"
import { promisify } from "node:util"
import { resolveSandboxRef } from "@/app/lib/openshellHost"
import { OPENSHELL_BIN, hostCommandEnv } from "@/app/lib/hostCommands"
import { HOST_PATH, NEMOCLAW_BIN, NEMOCLAW_CWD, OPENSHELL_BIN } from "@/app/lib/hostCommands"
const execFileAsync = promisify(execFile)
@@ -56,9 +54,14 @@ async function deleteSandbox(sandboxName: string) {
console.log(`[sandbox/delete] command:start sandbox=${sandboxName}`)
try {
const { stdout, stderr } = await execFileAsync(OPENSHELL_BIN, ["sandbox", "delete", sandboxName], {
env: hostCommandEnv({
env: {
...process.env,
PATH: HOST_PATH,
OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY || "nemoclaw",
}),
NO_COLOR: "1",
CLICOLOR: "0",
CLICOLOR_FORCE: "0",
},
timeout: 60000,
maxBuffer: 20 * 1024 * 1024,
})
@@ -80,50 +83,48 @@ function isSandboxNotFound(output: string) {
return /sandbox not found|status:\s*NotFound|not present in the live OpenShell gateway/i.test(output)
}
function isGatewayRecoverable(output: string) {
return /Unknown gateway|Deploy it first|transport error|tcp connect error|Connection refused|invalid peer certificate|BadSignature/i.test(output)
function isNemoClawSandboxNotRegistered(output: string, sandboxName: string) {
return new RegExp(`Unknown command:\\s*${sandboxName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`, "i").test(output)
}
const NEMOCLAW_REGISTRY_PATH = path.join(process.env.HOME || "/tmp", ".nemoclaw", "sandboxes.json")
async function cleanupNemoClawSandbox(sandboxName: string) {
const startedAt = Date.now()
console.log(`[sandbox/delete] nemoclaw-cleanup:start sandbox=${sandboxName}`)
try {
const raw = await readFile(NEMOCLAW_REGISTRY_PATH, "utf8")
const registry = JSON.parse(raw)
const sandboxes = registry && typeof registry === "object" && registry.sandboxes && typeof registry.sandboxes === "object"
? registry.sandboxes
: {}
const existed = Object.prototype.hasOwnProperty.call(sandboxes, sandboxName)
if (existed) delete sandboxes[sandboxName]
const remaining = Object.keys(sandboxes)
const defaultSandbox = registry.defaultSandbox === sandboxName
? remaining[0] || null
: (registry.defaultSandbox && sandboxes[registry.defaultSandbox] ? registry.defaultSandbox : remaining[0] || null)
const next = { ...registry, sandboxes, defaultSandbox }
await mkdir(path.dirname(NEMOCLAW_REGISTRY_PATH), { recursive: true, mode: 0o700 })
await writeFile(NEMOCLAW_REGISTRY_PATH, `${JSON.stringify(next, null, 2)}\n`, { mode: 0o600 })
return {
ok: true as const,
stdout: "",
stderr: "",
note: existed
? "Pruned NemoClaw registry entry without destroying the shared OpenShell gateway."
: "NemoClaw registry already had no matching sandbox entry.",
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error ?? "NemoClaw registry cleanup failed")
if (/ENOENT/.test(message)) {
const { stdout, stderr } = await execFileAsync(NEMOCLAW_BIN, [sandboxName, "destroy", "--yes"], {
cwd: NEMOCLAW_CWD,
env: {
...process.env,
PATH: HOST_PATH,
OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY || "nemoclaw",
NO_COLOR: "1",
CLICOLOR: "0",
CLICOLOR_FORCE: "0",
},
timeout: 90000,
maxBuffer: 20 * 1024 * 1024,
})
console.log(`[sandbox/delete] nemoclaw-cleanup:done sandbox=${sandboxName} elapsedMs=${elapsedMs(startedAt)}`)
return { ok: true as const, stdout: String(stdout).trim(), stderr: String(stderr).trim() }
} catch (error: any) {
const message = error instanceof Error ? error.message : String(error ?? "NemoClaw cleanup failed")
const stdout = String(error?.stdout || "").trim()
const stderr = String(error?.stderr || "").trim()
const output = [message, stdout, stderr].filter(Boolean).join("\n")
if (isNemoClawSandboxNotRegistered(output, sandboxName)) {
console.log(`[sandbox/delete] nemoclaw-cleanup:already-gone sandbox=${sandboxName} elapsedMs=${elapsedMs(startedAt)}`)
return {
ok: true as const,
stdout: "",
stderr: "",
note: "NemoClaw registry file does not exist; skipped registry cleanup.",
stdout,
stderr,
note: "NemoClaw registry already had no matching sandbox.",
}
}
console.log(`[sandbox/delete] nemoclaw-cleanup:error sandbox=${sandboxName} elapsedMs=${elapsedMs(startedAt)} message=${message}`)
return {
ok: false as const,
stdout: "",
stderr: "",
stdout,
stderr,
error: message,
}
}
@@ -178,6 +179,7 @@ export async function POST(request: Request) {
const result = await deleteSandbox(target.sandboxName)
const deleteOutput = [result.error, result.stdout, result.stderr].filter(Boolean).join("\n")
const openShellAlreadyGone = !result.ok && isSandboxNotFound(deleteOutput)
const cleanup = await cleanupNemoClawSandbox(target.sandboxName)
if (!result.ok && !openShellAlreadyGone) {
return NextResponse.json({
@@ -191,15 +193,10 @@ export async function POST(request: Request) {
error: result.error,
stdout: result.stdout,
stderr: result.stderr,
recoverableGateway: isGatewayRecoverable(deleteOutput),
note: isGatewayRecoverable(deleteOutput)
? "OpenShell gateway is unavailable or its trust state is broken. Repair gateway trust, then retry delete."
: "OpenShell delete failed before NemoClaw cleanup ran.",
}, { status: isGatewayRecoverable(deleteOutput) ? 503 : 500 })
nemoclaw: cleanup,
}, { status: 500 })
}
const cleanup = await cleanupNemoClawSandbox(target.sandboxName)
if (!cleanup.ok) {
return NextResponse.json({
ok: false,