Make sandbox restart non-destructive

This commit is contained in:
Codex
2026-04-28 21:13:22 +00:00
parent e86358c719
commit 2f91d8b706
3 changed files with 73 additions and 51 deletions
+65 -44
View File
@@ -1,12 +1,11 @@
import { NextResponse } from "next/server" import { NextResponse } from "next/server"
import { spawn } from "node:child_process" import { execFile } from "node:child_process"
import { applySandboxInferenceProfile } from "@/app/lib/sandboxInferenceApply" import { promisify } from "node:util"
import { getSandboxInferenceConfig } from "@/app/lib/sandboxInferenceStore" import { OPENSHELL_BIN, hostCommandEnv } from "@/app/lib/hostCommands"
import { inspectSandbox, resolveSandboxRef } from "@/app/lib/openshellHost" import { inspectSandbox, resolveSandboxRef } from "@/app/lib/openshellHost"
const DOCKER_BIN = process.env.DOCKER_BIN || "docker" const execFileAsync = promisify(execFile)
const OPENSHELL_CLUSTER_CONTAINER = process.env.OPENSHELL_CLUSTER_CONTAINER || "openshell-cluster-nemoclaw" const SANDBOX_DASHBOARD_REMOTE_PORT = Number.parseInt(process.env.OPENCLAW_SANDBOX_DASHBOARD_REMOTE_PORT || "18789", 10)
const OPENSHELL_SANDBOX_NAMESPACE = process.env.OPENSHELL_SANDBOX_NAMESPACE || "openshell"
function sleep(ms: number) { function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms)) return new Promise((resolve) => setTimeout(resolve, ms))
@@ -18,24 +17,25 @@ function validateSandboxName(value: string) {
return value return value
} }
async function runKubectl(args: string[]) { async function runOpenShell(args: string[], timeout = 30000) {
return await new Promise<{ stdout: string; stderr: string; code: number | null }>((resolve, reject) => { const { stdout, stderr } = await execFileAsync(OPENSHELL_BIN, args, {
const child = spawn(DOCKER_BIN, ["exec", OPENSHELL_CLUSTER_CONTAINER, "kubectl", ...args], { env: hostCommandEnv({
stdio: ["ignore", "pipe", "pipe"], OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY || "nemoclaw",
}) }),
let stdout = "" timeout,
let stderr = "" maxBuffer: 4 * 1024 * 1024,
child.stdout.on("data", (chunk) => { stdout += String(chunk) })
child.stderr.on("data", (chunk) => { stderr += String(chunk) })
child.on("error", reject)
child.on("close", (code) => resolve({ stdout: stdout.trim(), stderr: stderr.trim(), code }))
}) })
return { stdout: String(stdout).trim(), stderr: String(stderr).trim() }
}
async function runSandboxShell(sandboxName: string, script: string, timeout = 30000) {
return runOpenShell(["sandbox", "exec", "-n", sandboxName, "--", "sh", "-lc", script], timeout)
} }
async function waitForSandboxReady(sandboxName: string, timeoutMs: number, intervalMs: number) { async function waitForSandboxReady(sandboxName: string, timeoutMs: number, intervalMs: number) {
const startedAt = Date.now() const startedAt = Date.now()
let attempts = 0 let attempts = 0
let lastError: string | null = null let lastError = ""
while (Date.now() - startedAt < timeoutMs) { while (Date.now() - startedAt < timeoutMs) {
attempts += 1 attempts += 1
@@ -54,46 +54,67 @@ async function waitForSandboxReady(sandboxName: string, timeoutMs: number, inter
return { ready: false, attempts, elapsedMs: Date.now() - startedAt, lastError } return { ready: false, attempts, elapsedMs: Date.now() - startedAt, lastError }
} }
function restartOpenClawGatewayScript() {
return [
`port=${SANDBOX_DASHBOARD_REMOTE_PORT}`,
"for p in /proc/[0-9]*; do",
" cmd=$(tr '\\0' ' ' < \"$p/cmdline\" 2>/dev/null || true)",
" case \"$cmd\" in",
" *'openclaw gateway run'*) kill \"${p##*/}\" 2>/dev/null || true ;;",
" esac",
"done",
"sleep 1",
"if command -v openclaw >/dev/null 2>&1; then openclaw_bin=$(command -v openclaw);",
"elif [ -x /usr/local/bin/openclaw ]; then openclaw_bin=/usr/local/bin/openclaw;",
"else echo 'openclaw command not found in sandbox' >&2; exit 127; fi",
"nohup \"$openclaw_bin\" gateway run --allow-unconfigured --bind loopback --port \"$port\" >/tmp/gateway.log 2>&1 &",
"for i in 1 2 3 4 5 6 7 8 9 10; do",
" curl -fsS --max-time 2 \"http://127.0.0.1:$port/\" >/dev/null 2>&1 && exit 0",
" sleep 1",
"done",
"echo 'OpenClaw gateway did not answer after restart. Last log lines:' >&2",
"tail -40 /tmp/gateway.log >&2 2>/dev/null || true",
"exit 1",
].join("\n")
}
export async function POST( export async function POST(
_request: Request, _request: Request,
{ params }: { params: Promise<{ sandboxId: string }> } { params }: { params: Promise<{ sandboxId: string }> },
) { ) {
const startedAt = Date.now() const startedAt = Date.now()
try { try {
const { sandboxId } = await params const { sandboxId } = await params
const resolved = await resolveSandboxRef(sandboxId) const resolved = await resolveSandboxRef(sandboxId)
const sandboxName = validateSandboxName(resolved.name) const sandboxName = validateSandboxName(resolved.name)
const deleted = await runKubectl(["delete", "pod", sandboxName, "-n", OPENSHELL_SANDBOX_NAMESPACE, "--wait=false"]) const readiness = await waitForSandboxReady(sandboxName, 15000, 1000)
if (deleted.code !== 0 && !/not found/i.test(`${deleted.stdout}\n${deleted.stderr}`)) { if (!readiness.ready) {
throw new Error(deleted.stderr || deleted.stdout || "Failed to restart sandbox pod")
}
const readiness = await waitForSandboxReady(sandboxName, 90000, 2000)
let inferenceApply = null
if (readiness.ready) {
const inferenceConfig = await getSandboxInferenceConfig(resolved.id)
if (inferenceConfig.routes.some((route) => route.enabled)) {
inferenceApply = await applySandboxInferenceProfile(resolved.id, sandboxName)
}
}
return NextResponse.json({ return NextResponse.json({
ok: readiness.ready, ok: false,
restarted: readiness.ready, restarted: false,
sandboxId: resolved.id, sandboxId: resolved.id,
sandboxName, sandboxName,
deletion: deleted,
readiness, readiness,
inferenceApply,
elapsedMs: Date.now() - startedAt, elapsedMs: Date.now() - startedAt,
note: readiness.ready note: "Sandbox was not Ready, so the dashboard runtime was not restarted.",
? inferenceApply }, { status: 409 })
? "Sandbox pod restarted, OpenShell reports it Ready, and the saved inference profile was reapplied." }
: "Sandbox pod restarted and OpenShell reports it Ready."
: "Restart was requested, but OpenShell has not reported the sandbox Ready yet.", const runtime = await runSandboxShell(sandboxName, restartOpenClawGatewayScript(), 45000)
}, { status: readiness.ready ? 200 : 202 })
return NextResponse.json({
ok: true,
restarted: true,
restartMode: "openclaw-runtime",
sandboxId: resolved.id,
sandboxName,
readiness,
runtime,
elapsedMs: Date.now() - startedAt,
note: "OpenClaw runtime restarted inside the sandbox. The sandbox pod was not deleted.",
})
} catch (error) { } catch (error) {
const message = error instanceof Error ? error.message : "Failed to restart sandbox" const message = error instanceof Error ? error.message : "Failed to restart sandbox runtime"
return NextResponse.json({ ok: false, restarted: false, error: message }, { status: /required|invalid/.test(message) ? 400 : 500 }) return NextResponse.json({ ok: false, restarted: false, error: message }, { status: /required|invalid/.test(message) ? 400 : 500 })
} }
} }
+7 -7
View File
@@ -213,20 +213,20 @@ export default function SandboxList({
} }
} }
const restartSandbox = async () => { const restartSandboxRuntime = async () => {
if (!selectedSandbox || restartInProgress) return if (!selectedSandbox || restartInProgress) return
try { try {
setRestartInProgress(true) setRestartInProgress(true)
setDashboardMessage(`Restarting sandbox ${selectedSandbox.name}...`) setDashboardMessage(`Restarting OpenClaw runtime for ${selectedSandbox.name}...`)
const response = await fetch(`/api/sandbox/${encodeURIComponent(selectedSandbox.id)}/restart`, { const response = await fetch(`/api/sandbox/${encodeURIComponent(selectedSandbox.id)}/restart`, {
method: 'POST', method: 'POST',
}) })
const data = await response.json() const data = await response.json()
if (!response.ok) throw new Error(data.error || data.note || 'Failed to restart sandbox') if (!response.ok) throw new Error(data.error || data.note || 'Failed to restart OpenClaw runtime')
setDashboardMessage(data.note || `Sandbox ${selectedSandbox.name} restarted.`) setDashboardMessage(data.note || `OpenClaw runtime restarted for ${selectedSandbox.name}.`)
await onInventoryRefresh() await onInventoryRefresh()
} catch (error) { } catch (error) {
setDashboardMessage(error instanceof Error ? error.message : 'Failed to restart sandbox.') setDashboardMessage(error instanceof Error ? error.message : 'Failed to restart OpenClaw runtime.')
} finally { } finally {
setRestartInProgress(false) setRestartInProgress(false)
} }
@@ -538,11 +538,11 @@ export default function SandboxList({
Start OpenClaw Gateway Dashboard Start OpenClaw Gateway Dashboard
</button> </button>
<button <button
onClick={restartSandbox} onClick={restartSandboxRuntime}
disabled={restartInProgress} disabled={restartInProgress}
className="action-button px-3 py-2" className="action-button px-3 py-2"
> >
{restartInProgress ? 'Restarting Sandbox...' : 'Restart Sandbox'} {restartInProgress ? 'Restarting Runtime...' : 'Restart OpenClaw Runtime'}
</button> </button>
<button <button
onClick={async () => { onClick={async () => {
+4 -3
View File
@@ -105,9 +105,10 @@ assert.match(sandboxInferencePanelSource, /Add Route/, 'selected sandbox UI must
assert.match(sandboxInferenceApplyLibSource, /\/sandbox\/\.openclaw\/openclaw\.json/, 'live apply must patch OpenClaw config inside the running sandbox') assert.match(sandboxInferenceApplyLibSource, /\/sandbox\/\.openclaw\/openclaw\.json/, 'live apply must patch OpenClaw config inside the running sandbox')
assert.match(sandboxInferenceApplyLibSource, /sha256sum \/sandbox\/\.openclaw\/openclaw\.json > \/sandbox\/\.openclaw\/\.config-hash/, 'live apply must refresh the immutable OpenClaw config hash') assert.match(sandboxInferenceApplyLibSource, /sha256sum \/sandbox\/\.openclaw\/openclaw\.json > \/sandbox\/\.openclaw\/\.config-hash/, 'live apply must refresh the immutable OpenClaw config hash')
assert.match(sandboxInferenceApplyLibSource, /\["inference", "set", "--no-verify", "--provider", primary\.provider, "--model", primary\.model\]/, 'live apply must point OpenShell inference at the primary route') assert.match(sandboxInferenceApplyLibSource, /\["inference", "set", "--no-verify", "--provider", primary\.provider, "--model", primary\.model\]/, 'live apply must point OpenShell inference at the primary route')
assert.match(sandboxRestartRouteSource, /\["delete", "pod", sandboxName, "-n", OPENSHELL_SANDBOX_NAMESPACE, "--wait=false"\]/, 'restart route must restart the backing sandbox pod') assert.doesNotMatch(sandboxRestartRouteSource, /delete", "pod"|kubectl delete/, 'restart route must not delete the backing sandbox pod')
assert.match(sandboxRestartRouteSource, /waitForSandboxReady\(sandboxName, 90000, 2000\)/, 'restart route must wait for OpenShell to report the sandbox ready') assert.match(sandboxRestartRouteSource, /restartMode: "openclaw-runtime"/, 'restart route must restart only the in-sandbox OpenClaw runtime')
assert.match(sandboxListSource, /Restart Sandbox/, 'sandbox details must expose a restart sandbox button') assert.match(sandboxRestartRouteSource, /waitForSandboxReady\(sandboxName, 15000, 1000\)/, 'restart route must verify OpenShell reports the sandbox ready before runtime restart')
assert.match(sandboxListSource, /Restart OpenClaw Runtime/, 'sandbox details must expose a scoped runtime restart button')
assert.match(sandboxListSource, /\/api\/sandbox\/\$\{encodeURIComponent\(selectedSandbox\.id\)\}\/restart/, 'restart button must call the sandbox restart endpoint') assert.match(sandboxListSource, /\/api\/sandbox\/\$\{encodeURIComponent\(selectedSandbox\.id\)\}\/restart/, 'restart button must call the sandbox restart endpoint')
assert.doesNotMatch(sandboxListSource, /setInterval\(loadPermissionFeeds/, 'sandbox list must not poll every sandbox permission feed') assert.doesNotMatch(sandboxListSource, /setInterval\(loadPermissionFeeds/, 'sandbox list must not poll every sandbox permission feed')
assert.match(sandboxListSource, /loadPermissionFeed\(selectedSandbox\)/, 'policy drawer must load the selected sandbox permission feed on demand') assert.match(sandboxListSource, /loadPermissionFeed\(selectedSandbox\)/, 'policy drawer must load the selected sandbox permission feed on demand')