Make sandbox restart non-destructive

This commit is contained in:
Codex
2026-04-28 21:13:22 +00:00
parent e86358c719
commit 2f91d8b706
3 changed files with 73 additions and 51 deletions
+65 -44
View File
@@ -1,12 +1,11 @@
import { NextResponse } from "next/server"
import { spawn } from "node:child_process"
import { applySandboxInferenceProfile } from "@/app/lib/sandboxInferenceApply"
import { getSandboxInferenceConfig } from "@/app/lib/sandboxInferenceStore"
import { execFile } from "node:child_process"
import { promisify } from "node:util"
import { OPENSHELL_BIN, hostCommandEnv } from "@/app/lib/hostCommands"
import { inspectSandbox, resolveSandboxRef } from "@/app/lib/openshellHost"
const DOCKER_BIN = process.env.DOCKER_BIN || "docker"
const OPENSHELL_CLUSTER_CONTAINER = process.env.OPENSHELL_CLUSTER_CONTAINER || "openshell-cluster-nemoclaw"
const OPENSHELL_SANDBOX_NAMESPACE = process.env.OPENSHELL_SANDBOX_NAMESPACE || "openshell"
const execFileAsync = promisify(execFile)
const SANDBOX_DASHBOARD_REMOTE_PORT = Number.parseInt(process.env.OPENCLAW_SANDBOX_DASHBOARD_REMOTE_PORT || "18789", 10)
function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms))
@@ -18,24 +17,25 @@ function validateSandboxName(value: string) {
return value
}
async function runKubectl(args: string[]) {
return await new Promise<{ stdout: string; stderr: string; code: number | null }>((resolve, reject) => {
const child = spawn(DOCKER_BIN, ["exec", OPENSHELL_CLUSTER_CONTAINER, "kubectl", ...args], {
stdio: ["ignore", "pipe", "pipe"],
})
let stdout = ""
let stderr = ""
child.stdout.on("data", (chunk) => { stdout += String(chunk) })
child.stderr.on("data", (chunk) => { stderr += String(chunk) })
child.on("error", reject)
child.on("close", (code) => resolve({ stdout: stdout.trim(), stderr: stderr.trim(), code }))
async function runOpenShell(args: string[], timeout = 30000) {
const { stdout, stderr } = await execFileAsync(OPENSHELL_BIN, args, {
env: hostCommandEnv({
OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY || "nemoclaw",
}),
timeout,
maxBuffer: 4 * 1024 * 1024,
})
return { stdout: String(stdout).trim(), stderr: String(stderr).trim() }
}
async function runSandboxShell(sandboxName: string, script: string, timeout = 30000) {
return runOpenShell(["sandbox", "exec", "-n", sandboxName, "--", "sh", "-lc", script], timeout)
}
async function waitForSandboxReady(sandboxName: string, timeoutMs: number, intervalMs: number) {
const startedAt = Date.now()
let attempts = 0
let lastError: string | null = null
let lastError = ""
while (Date.now() - startedAt < timeoutMs) {
attempts += 1
@@ -54,46 +54,67 @@ async function waitForSandboxReady(sandboxName: string, timeoutMs: number, inter
return { ready: false, attempts, elapsedMs: Date.now() - startedAt, lastError }
}
function restartOpenClawGatewayScript() {
return [
`port=${SANDBOX_DASHBOARD_REMOTE_PORT}`,
"for p in /proc/[0-9]*; do",
" cmd=$(tr '\\0' ' ' < \"$p/cmdline\" 2>/dev/null || true)",
" case \"$cmd\" in",
" *'openclaw gateway run'*) kill \"${p##*/}\" 2>/dev/null || true ;;",
" esac",
"done",
"sleep 1",
"if command -v openclaw >/dev/null 2>&1; then openclaw_bin=$(command -v openclaw);",
"elif [ -x /usr/local/bin/openclaw ]; then openclaw_bin=/usr/local/bin/openclaw;",
"else echo 'openclaw command not found in sandbox' >&2; exit 127; fi",
"nohup \"$openclaw_bin\" gateway run --allow-unconfigured --bind loopback --port \"$port\" >/tmp/gateway.log 2>&1 &",
"for i in 1 2 3 4 5 6 7 8 9 10; do",
" curl -fsS --max-time 2 \"http://127.0.0.1:$port/\" >/dev/null 2>&1 && exit 0",
" sleep 1",
"done",
"echo 'OpenClaw gateway did not answer after restart. Last log lines:' >&2",
"tail -40 /tmp/gateway.log >&2 2>/dev/null || true",
"exit 1",
].join("\n")
}
export async function POST(
_request: Request,
{ params }: { params: Promise<{ sandboxId: string }> }
{ params }: { params: Promise<{ sandboxId: string }> },
) {
const startedAt = Date.now()
try {
const { sandboxId } = await params
const resolved = await resolveSandboxRef(sandboxId)
const sandboxName = validateSandboxName(resolved.name)
const deleted = await runKubectl(["delete", "pod", sandboxName, "-n", OPENSHELL_SANDBOX_NAMESPACE, "--wait=false"])
if (deleted.code !== 0 && !/not found/i.test(`${deleted.stdout}\n${deleted.stderr}`)) {
throw new Error(deleted.stderr || deleted.stdout || "Failed to restart sandbox pod")
}
const readiness = await waitForSandboxReady(sandboxName, 90000, 2000)
let inferenceApply = null
if (readiness.ready) {
const inferenceConfig = await getSandboxInferenceConfig(resolved.id)
if (inferenceConfig.routes.some((route) => route.enabled)) {
inferenceApply = await applySandboxInferenceProfile(resolved.id, sandboxName)
}
}
const readiness = await waitForSandboxReady(sandboxName, 15000, 1000)
if (!readiness.ready) {
return NextResponse.json({
ok: readiness.ready,
restarted: readiness.ready,
ok: false,
restarted: false,
sandboxId: resolved.id,
sandboxName,
deletion: deleted,
readiness,
inferenceApply,
elapsedMs: Date.now() - startedAt,
note: readiness.ready
? inferenceApply
? "Sandbox pod restarted, OpenShell reports it Ready, and the saved inference profile was reapplied."
: "Sandbox pod restarted and OpenShell reports it Ready."
: "Restart was requested, but OpenShell has not reported the sandbox Ready yet.",
}, { status: readiness.ready ? 200 : 202 })
note: "Sandbox was not Ready, so the dashboard runtime was not restarted.",
}, { status: 409 })
}
const runtime = await runSandboxShell(sandboxName, restartOpenClawGatewayScript(), 45000)
return NextResponse.json({
ok: true,
restarted: true,
restartMode: "openclaw-runtime",
sandboxId: resolved.id,
sandboxName,
readiness,
runtime,
elapsedMs: Date.now() - startedAt,
note: "OpenClaw runtime restarted inside the sandbox. The sandbox pod was not deleted.",
})
} catch (error) {
const message = error instanceof Error ? error.message : "Failed to restart sandbox"
const message = error instanceof Error ? error.message : "Failed to restart sandbox runtime"
return NextResponse.json({ ok: false, restarted: false, error: message }, { status: /required|invalid/.test(message) ? 400 : 500 })
}
}
+7 -7
View File
@@ -213,20 +213,20 @@ export default function SandboxList({
}
}
const restartSandbox = async () => {
const restartSandboxRuntime = async () => {
if (!selectedSandbox || restartInProgress) return
try {
setRestartInProgress(true)
setDashboardMessage(`Restarting sandbox ${selectedSandbox.name}...`)
setDashboardMessage(`Restarting OpenClaw runtime for ${selectedSandbox.name}...`)
const response = await fetch(`/api/sandbox/${encodeURIComponent(selectedSandbox.id)}/restart`, {
method: 'POST',
})
const data = await response.json()
if (!response.ok) throw new Error(data.error || data.note || 'Failed to restart sandbox')
setDashboardMessage(data.note || `Sandbox ${selectedSandbox.name} restarted.`)
if (!response.ok) throw new Error(data.error || data.note || 'Failed to restart OpenClaw runtime')
setDashboardMessage(data.note || `OpenClaw runtime restarted for ${selectedSandbox.name}.`)
await onInventoryRefresh()
} catch (error) {
setDashboardMessage(error instanceof Error ? error.message : 'Failed to restart sandbox.')
setDashboardMessage(error instanceof Error ? error.message : 'Failed to restart OpenClaw runtime.')
} finally {
setRestartInProgress(false)
}
@@ -538,11 +538,11 @@ export default function SandboxList({
Start OpenClaw Gateway Dashboard
</button>
<button
onClick={restartSandbox}
onClick={restartSandboxRuntime}
disabled={restartInProgress}
className="action-button px-3 py-2"
>
{restartInProgress ? 'Restarting Sandbox...' : 'Restart Sandbox'}
{restartInProgress ? 'Restarting Runtime...' : 'Restart OpenClaw Runtime'}
</button>
<button
onClick={async () => {
+4 -3
View File
@@ -105,9 +105,10 @@ assert.match(sandboxInferencePanelSource, /Add Route/, 'selected sandbox UI must
assert.match(sandboxInferenceApplyLibSource, /\/sandbox\/\.openclaw\/openclaw\.json/, 'live apply must patch OpenClaw config inside the running sandbox')
assert.match(sandboxInferenceApplyLibSource, /sha256sum \/sandbox\/\.openclaw\/openclaw\.json > \/sandbox\/\.openclaw\/\.config-hash/, 'live apply must refresh the immutable OpenClaw config hash')
assert.match(sandboxInferenceApplyLibSource, /\["inference", "set", "--no-verify", "--provider", primary\.provider, "--model", primary\.model\]/, 'live apply must point OpenShell inference at the primary route')
assert.match(sandboxRestartRouteSource, /\["delete", "pod", sandboxName, "-n", OPENSHELL_SANDBOX_NAMESPACE, "--wait=false"\]/, 'restart route must restart the backing sandbox pod')
assert.match(sandboxRestartRouteSource, /waitForSandboxReady\(sandboxName, 90000, 2000\)/, 'restart route must wait for OpenShell to report the sandbox ready')
assert.match(sandboxListSource, /Restart Sandbox/, 'sandbox details must expose a restart sandbox button')
assert.doesNotMatch(sandboxRestartRouteSource, /delete", "pod"|kubectl delete/, 'restart route must not delete the backing sandbox pod')
assert.match(sandboxRestartRouteSource, /restartMode: "openclaw-runtime"/, 'restart route must restart only the in-sandbox OpenClaw runtime')
assert.match(sandboxRestartRouteSource, /waitForSandboxReady\(sandboxName, 15000, 1000\)/, 'restart route must verify OpenShell reports the sandbox ready before runtime restart')
assert.match(sandboxListSource, /Restart OpenClaw Runtime/, 'sandbox details must expose a scoped runtime restart button')
assert.match(sandboxListSource, /\/api\/sandbox\/\$\{encodeURIComponent\(selectedSandbox\.id\)\}\/restart/, 'restart button must call the sandbox restart endpoint')
assert.doesNotMatch(sandboxListSource, /setInterval\(loadPermissionFeeds/, 'sandbox list must not poll every sandbox permission feed')
assert.match(sandboxListSource, /loadPermissionFeed\(selectedSandbox\)/, 'policy drawer must load the selected sandbox permission feed on demand')