Roll back sandbox create destroy workflow to Monday
This commit is contained in:
@@ -2,7 +2,6 @@ import { NextResponse } from "next/server"
|
||||
import { execFile, spawn } from "node:child_process"
|
||||
import { promisify } from "node:util"
|
||||
import { inspectSandbox } from "@/app/lib/openshellHost"
|
||||
import { repairOpenClawWorkspacePermissions, stabilizeOpenClawGatewayConfig } from "@/app/lib/sandboxPrivilegedFiles"
|
||||
import {
|
||||
commandExists,
|
||||
HOST_PATH,
|
||||
@@ -58,60 +57,6 @@ type NemoClawCreateCommand = {
|
||||
mode: "cli-onboard" | "legacy-setup"
|
||||
}
|
||||
|
||||
type CreateInferenceMode = "auto" | "vllm" | "nim"
|
||||
|
||||
type CreateInferenceSettings = {
|
||||
mode: CreateInferenceMode
|
||||
model: string
|
||||
hasApiKey: boolean
|
||||
envSummary: string[]
|
||||
}
|
||||
|
||||
function parseCreateInferenceSettings(body: any): CreateInferenceSettings {
|
||||
const raw = body?.createInference && typeof body.createInference === "object" ? body.createInference : {}
|
||||
const requestedMode = typeof raw.mode === "string" ? raw.mode : "auto"
|
||||
const mode: CreateInferenceMode = requestedMode === "vllm" || requestedMode === "nim" ? requestedMode : "auto"
|
||||
const model = typeof raw.model === "string" ? raw.model.trim() : ""
|
||||
const apiKey = typeof raw.apiKey === "string" ? raw.apiKey.trim() : ""
|
||||
|
||||
return {
|
||||
mode,
|
||||
model,
|
||||
hasApiKey: Boolean(apiKey),
|
||||
envSummary: [],
|
||||
}
|
||||
}
|
||||
|
||||
function applyCreateInferenceEnv(env: NodeJS.ProcessEnv, settings: CreateInferenceSettings, body: any) {
|
||||
if (settings.mode === "auto") return settings
|
||||
|
||||
env.NEMOCLAW_EXPERIMENTAL = "1"
|
||||
settings.envSummary.push("NEMOCLAW_EXPERIMENTAL=1")
|
||||
|
||||
if (settings.mode === "vllm") {
|
||||
env.NEMOCLAW_PROVIDER = "vllm"
|
||||
settings.envSummary.push("NEMOCLAW_PROVIDER=vllm")
|
||||
}
|
||||
|
||||
if (settings.mode === "nim") {
|
||||
env.NEMOCLAW_PROVIDER = "nim-local"
|
||||
settings.envSummary.push("NEMOCLAW_PROVIDER=nim-local")
|
||||
const apiKey = typeof body?.createInference?.apiKey === "string" ? body.createInference.apiKey.trim() : ""
|
||||
if (apiKey) {
|
||||
env.NVIDIA_API_KEY = apiKey
|
||||
env.NEMOCLAW_PROVIDER_KEY = apiKey
|
||||
settings.envSummary.push("NVIDIA_API_KEY=<provided>", "NEMOCLAW_PROVIDER_KEY=<provided>")
|
||||
}
|
||||
}
|
||||
|
||||
if (settings.model) {
|
||||
env.NEMOCLAW_MODEL = settings.model
|
||||
settings.envSummary.push(`NEMOCLAW_MODEL=${settings.model}`)
|
||||
}
|
||||
|
||||
return settings
|
||||
}
|
||||
|
||||
function buildNemoClawCreateCommand(): NemoClawCreateCommand {
|
||||
if (NEMOCLAW_BIN && commandExists(NEMOCLAW_BIN)) {
|
||||
const args = ["onboard", "--non-interactive", "--recreate-sandbox", "--yes-i-accept-third-party-software"]
|
||||
@@ -386,8 +331,7 @@ export async function POST(request: Request) {
|
||||
const sandboxName = validateSandboxName(typeof body?.sandboxName === "string" ? body.sandboxName.trim() : "")
|
||||
const policy = body?.policy && typeof body.policy === "object" ? body.policy : null
|
||||
const enableTailscale = Boolean(body?.enableTailscale)
|
||||
const createInference = parseCreateInferenceSettings(body)
|
||||
console.log(`[sandbox/create] request:parsed sandbox=${sandboxName} blueprint=${blueprint} enableTailscale=${enableTailscale} inferenceMode=${createInference.mode} elapsedMs=${elapsedMs(requestStartedAt)}`)
|
||||
console.log(`[sandbox/create] request:parsed sandbox=${sandboxName} blueprint=${blueprint} enableTailscale=${enableTailscale} elapsedMs=${elapsedMs(requestStartedAt)}`)
|
||||
|
||||
if (!blueprint) {
|
||||
return NextResponse.json({ ok: false, error: "blueprint is required" }, { status: 400 })
|
||||
@@ -409,8 +353,6 @@ export async function POST(request: Request) {
|
||||
env.NVIDIA_API_KEY = env.NVIDIA_API_KEY || "optional-local-mode"
|
||||
}
|
||||
|
||||
applyCreateInferenceEnv(env, createInference, body)
|
||||
|
||||
const result = await runCommand(
|
||||
createCommand.file,
|
||||
createCommand.mode === "legacy-setup" ? [...createCommand.args, sandboxName] : createCommand.args,
|
||||
@@ -427,7 +369,6 @@ export async function POST(request: Request) {
|
||||
blueprint,
|
||||
sandboxName,
|
||||
enableTailscale,
|
||||
createInference,
|
||||
}, { status: 500 })
|
||||
}
|
||||
|
||||
@@ -438,16 +379,6 @@ export async function POST(request: Request) {
|
||||
error: "Sandbox readiness polling produced no verification result.",
|
||||
}
|
||||
const created = readiness.verified
|
||||
const gatewayConfigRepair = created ? await stabilizeOpenClawGatewayConfig(sandboxName).catch((error) => ({
|
||||
sandboxName,
|
||||
path: "/sandbox/.openclaw/openclaw.json",
|
||||
error: error instanceof Error ? error.message : "Failed to stabilize OpenClaw gateway config",
|
||||
})) : null
|
||||
const workspaceRepair = created ? await repairOpenClawWorkspacePermissions(sandboxName).catch((error) => ({
|
||||
sandboxName,
|
||||
path: "/sandbox/.openclaw/workspace",
|
||||
error: error instanceof Error ? error.message : "Failed to repair OpenClaw workspace permissions",
|
||||
})) : null
|
||||
const deviceApproval = created ? await approveOpenClawDeviceRequests(sandboxName) : null
|
||||
console.log(
|
||||
`[sandbox/create] request:complete sandbox=${sandboxName} created=${created} readinessAttempts=${readiness.attempts} deviceApproval=${deviceApproval?.approved ?? false} elapsedMs=${elapsedMs(requestStartedAt)}`,
|
||||
@@ -462,15 +393,12 @@ export async function POST(request: Request) {
|
||||
verification,
|
||||
mode: "nemoclaw-blueprint",
|
||||
enableTailscale,
|
||||
createInference,
|
||||
createCommand,
|
||||
hostPath: HOST_PATH,
|
||||
readiness: {
|
||||
attempts: readiness.attempts,
|
||||
elapsedMs: readiness.elapsedMs,
|
||||
},
|
||||
workspaceRepair,
|
||||
gatewayConfigRepair,
|
||||
deviceApproval,
|
||||
stdout: result.stdout,
|
||||
stderr: result.stderr,
|
||||
@@ -479,10 +407,6 @@ export async function POST(request: Request) {
|
||||
enableTailscale
|
||||
? "NemoClaw blueprint workflow completed with Tailscale-enabled prerequisites. Existing healthy OpenShell gateways are reused before any new gateway start is attempted."
|
||||
: "NemoClaw blueprint workflow completed in local/default mode. Existing healthy OpenShell gateways are reused before any new gateway start is attempted.",
|
||||
workspaceRepair && "note" in workspaceRepair ? workspaceRepair.note : false,
|
||||
workspaceRepair && "error" in workspaceRepair ? `OpenClaw workspace permission repair failed: ${workspaceRepair.error}` : false,
|
||||
gatewayConfigRepair && "note" in gatewayConfigRepair ? gatewayConfigRepair.note : false,
|
||||
gatewayConfigRepair && "error" in gatewayConfigRepair ? `OpenClaw gateway config repair failed: ${gatewayConfigRepair.error}` : false,
|
||||
deviceApproval?.note,
|
||||
)
|
||||
: "Blueprint command reported success, but the sandbox never reached a ready verification state afterward.",
|
||||
@@ -517,16 +441,6 @@ export async function POST(request: Request) {
|
||||
error: "Sandbox readiness polling produced no verification result.",
|
||||
}
|
||||
const created = readiness.verified
|
||||
const gatewayConfigRepair = created ? await stabilizeOpenClawGatewayConfig(sandboxName).catch((error) => ({
|
||||
sandboxName,
|
||||
path: "/sandbox/.openclaw/openclaw.json",
|
||||
error: error instanceof Error ? error.message : "Failed to stabilize OpenClaw gateway config",
|
||||
})) : null
|
||||
const workspaceRepair = created ? await repairOpenClawWorkspacePermissions(sandboxName).catch((error) => ({
|
||||
sandboxName,
|
||||
path: "/sandbox/.openclaw/workspace",
|
||||
error: error instanceof Error ? error.message : "Failed to repair OpenClaw workspace permissions",
|
||||
})) : null
|
||||
const deviceApproval = created ? await approveOpenClawDeviceRequests(sandboxName) : null
|
||||
const policyPrepared = Boolean(policy)
|
||||
console.log(
|
||||
@@ -553,8 +467,6 @@ export async function POST(request: Request) {
|
||||
attempts: readiness.attempts,
|
||||
elapsedMs: readiness.elapsedMs,
|
||||
},
|
||||
workspaceRepair,
|
||||
gatewayConfigRepair,
|
||||
deviceApproval,
|
||||
policyPrepared,
|
||||
note: created
|
||||
@@ -564,10 +476,6 @@ export async function POST(request: Request) {
|
||||
: (policyPrepared
|
||||
? "Custom sandbox created. Policy draft is prepared, but applying it live should be a follow-up action."
|
||||
: "Custom sandbox created."),
|
||||
workspaceRepair && "note" in workspaceRepair ? workspaceRepair.note : false,
|
||||
workspaceRepair && "error" in workspaceRepair ? `OpenClaw workspace permission repair failed: ${workspaceRepair.error}` : false,
|
||||
gatewayConfigRepair && "note" in gatewayConfigRepair ? gatewayConfigRepair.note : false,
|
||||
gatewayConfigRepair && "error" in gatewayConfigRepair ? `OpenClaw gateway config repair failed: ${gatewayConfigRepair.error}` : false,
|
||||
deviceApproval?.note,
|
||||
)
|
||||
: "Create command started, but the sandbox never reached a ready verification state afterward.",
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
import { NextResponse } from "next/server"
|
||||
import { execFile } from "node:child_process"
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises"
|
||||
import path from "node:path"
|
||||
import { promisify } from "node:util"
|
||||
import { resolveSandboxRef } from "@/app/lib/openshellHost"
|
||||
import { OPENSHELL_BIN, hostCommandEnv } from "@/app/lib/hostCommands"
|
||||
import { HOST_PATH, NEMOCLAW_BIN, NEMOCLAW_CWD, OPENSHELL_BIN } from "@/app/lib/hostCommands"
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
|
||||
@@ -56,9 +54,14 @@ async function deleteSandbox(sandboxName: string) {
|
||||
console.log(`[sandbox/delete] command:start sandbox=${sandboxName}`)
|
||||
try {
|
||||
const { stdout, stderr } = await execFileAsync(OPENSHELL_BIN, ["sandbox", "delete", sandboxName], {
|
||||
env: hostCommandEnv({
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: HOST_PATH,
|
||||
OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY || "nemoclaw",
|
||||
}),
|
||||
NO_COLOR: "1",
|
||||
CLICOLOR: "0",
|
||||
CLICOLOR_FORCE: "0",
|
||||
},
|
||||
timeout: 60000,
|
||||
maxBuffer: 20 * 1024 * 1024,
|
||||
})
|
||||
@@ -80,50 +83,48 @@ function isSandboxNotFound(output: string) {
|
||||
return /sandbox not found|status:\s*NotFound|not present in the live OpenShell gateway/i.test(output)
|
||||
}
|
||||
|
||||
function isGatewayRecoverable(output: string) {
|
||||
return /Unknown gateway|Deploy it first|transport error|tcp connect error|Connection refused|invalid peer certificate|BadSignature/i.test(output)
|
||||
function isNemoClawSandboxNotRegistered(output: string, sandboxName: string) {
|
||||
return new RegExp(`Unknown command:\\s*${sandboxName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`, "i").test(output)
|
||||
}
|
||||
|
||||
const NEMOCLAW_REGISTRY_PATH = path.join(process.env.HOME || "/tmp", ".nemoclaw", "sandboxes.json")
|
||||
|
||||
async function cleanupNemoClawSandbox(sandboxName: string) {
|
||||
const startedAt = Date.now()
|
||||
console.log(`[sandbox/delete] nemoclaw-cleanup:start sandbox=${sandboxName}`)
|
||||
try {
|
||||
const raw = await readFile(NEMOCLAW_REGISTRY_PATH, "utf8")
|
||||
const registry = JSON.parse(raw)
|
||||
const sandboxes = registry && typeof registry === "object" && registry.sandboxes && typeof registry.sandboxes === "object"
|
||||
? registry.sandboxes
|
||||
: {}
|
||||
const existed = Object.prototype.hasOwnProperty.call(sandboxes, sandboxName)
|
||||
if (existed) delete sandboxes[sandboxName]
|
||||
const remaining = Object.keys(sandboxes)
|
||||
const defaultSandbox = registry.defaultSandbox === sandboxName
|
||||
? remaining[0] || null
|
||||
: (registry.defaultSandbox && sandboxes[registry.defaultSandbox] ? registry.defaultSandbox : remaining[0] || null)
|
||||
const next = { ...registry, sandboxes, defaultSandbox }
|
||||
await mkdir(path.dirname(NEMOCLAW_REGISTRY_PATH), { recursive: true, mode: 0o700 })
|
||||
await writeFile(NEMOCLAW_REGISTRY_PATH, `${JSON.stringify(next, null, 2)}\n`, { mode: 0o600 })
|
||||
return {
|
||||
ok: true as const,
|
||||
stdout: "",
|
||||
stderr: "",
|
||||
note: existed
|
||||
? "Pruned NemoClaw registry entry without destroying the shared OpenShell gateway."
|
||||
: "NemoClaw registry already had no matching sandbox entry.",
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error ?? "NemoClaw registry cleanup failed")
|
||||
if (/ENOENT/.test(message)) {
|
||||
const { stdout, stderr } = await execFileAsync(NEMOCLAW_BIN, [sandboxName, "destroy", "--yes"], {
|
||||
cwd: NEMOCLAW_CWD,
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: HOST_PATH,
|
||||
OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY || "nemoclaw",
|
||||
NO_COLOR: "1",
|
||||
CLICOLOR: "0",
|
||||
CLICOLOR_FORCE: "0",
|
||||
},
|
||||
timeout: 90000,
|
||||
maxBuffer: 20 * 1024 * 1024,
|
||||
})
|
||||
console.log(`[sandbox/delete] nemoclaw-cleanup:done sandbox=${sandboxName} elapsedMs=${elapsedMs(startedAt)}`)
|
||||
return { ok: true as const, stdout: String(stdout).trim(), stderr: String(stderr).trim() }
|
||||
} catch (error: any) {
|
||||
const message = error instanceof Error ? error.message : String(error ?? "NemoClaw cleanup failed")
|
||||
const stdout = String(error?.stdout || "").trim()
|
||||
const stderr = String(error?.stderr || "").trim()
|
||||
const output = [message, stdout, stderr].filter(Boolean).join("\n")
|
||||
if (isNemoClawSandboxNotRegistered(output, sandboxName)) {
|
||||
console.log(`[sandbox/delete] nemoclaw-cleanup:already-gone sandbox=${sandboxName} elapsedMs=${elapsedMs(startedAt)}`)
|
||||
return {
|
||||
ok: true as const,
|
||||
stdout: "",
|
||||
stderr: "",
|
||||
note: "NemoClaw registry file does not exist; skipped registry cleanup.",
|
||||
stdout,
|
||||
stderr,
|
||||
note: "NemoClaw registry already had no matching sandbox.",
|
||||
}
|
||||
}
|
||||
console.log(`[sandbox/delete] nemoclaw-cleanup:error sandbox=${sandboxName} elapsedMs=${elapsedMs(startedAt)} message=${message}`)
|
||||
return {
|
||||
ok: false as const,
|
||||
stdout: "",
|
||||
stderr: "",
|
||||
stdout,
|
||||
stderr,
|
||||
error: message,
|
||||
}
|
||||
}
|
||||
@@ -178,6 +179,7 @@ export async function POST(request: Request) {
|
||||
const result = await deleteSandbox(target.sandboxName)
|
||||
const deleteOutput = [result.error, result.stdout, result.stderr].filter(Boolean).join("\n")
|
||||
const openShellAlreadyGone = !result.ok && isSandboxNotFound(deleteOutput)
|
||||
const cleanup = await cleanupNemoClawSandbox(target.sandboxName)
|
||||
|
||||
if (!result.ok && !openShellAlreadyGone) {
|
||||
return NextResponse.json({
|
||||
@@ -191,15 +193,10 @@ export async function POST(request: Request) {
|
||||
error: result.error,
|
||||
stdout: result.stdout,
|
||||
stderr: result.stderr,
|
||||
recoverableGateway: isGatewayRecoverable(deleteOutput),
|
||||
note: isGatewayRecoverable(deleteOutput)
|
||||
? "OpenShell gateway is unavailable or its trust state is broken. Repair gateway trust, then retry delete."
|
||||
: "OpenShell delete failed before NemoClaw cleanup ran.",
|
||||
}, { status: isGatewayRecoverable(deleteOutput) ? 503 : 500 })
|
||||
nemoclaw: cleanup,
|
||||
}, { status: 500 })
|
||||
}
|
||||
|
||||
const cleanup = await cleanupNemoClawSandbox(target.sandboxName)
|
||||
|
||||
if (!cleanup.ok) {
|
||||
return NextResponse.json({
|
||||
ok: false,
|
||||
|
||||
Reference in New Issue
Block a user