Make sandbox restart non-destructive
This commit is contained in:
@@ -1,12 +1,11 @@
|
|||||||
import { NextResponse } from "next/server"
|
import { NextResponse } from "next/server"
|
||||||
import { spawn } from "node:child_process"
|
import { execFile } from "node:child_process"
|
||||||
import { applySandboxInferenceProfile } from "@/app/lib/sandboxInferenceApply"
|
import { promisify } from "node:util"
|
||||||
import { getSandboxInferenceConfig } from "@/app/lib/sandboxInferenceStore"
|
import { OPENSHELL_BIN, hostCommandEnv } from "@/app/lib/hostCommands"
|
||||||
import { inspectSandbox, resolveSandboxRef } from "@/app/lib/openshellHost"
|
import { inspectSandbox, resolveSandboxRef } from "@/app/lib/openshellHost"
|
||||||
|
|
||||||
const DOCKER_BIN = process.env.DOCKER_BIN || "docker"
|
const execFileAsync = promisify(execFile)
|
||||||
const OPENSHELL_CLUSTER_CONTAINER = process.env.OPENSHELL_CLUSTER_CONTAINER || "openshell-cluster-nemoclaw"
|
const SANDBOX_DASHBOARD_REMOTE_PORT = Number.parseInt(process.env.OPENCLAW_SANDBOX_DASHBOARD_REMOTE_PORT || "18789", 10)
|
||||||
const OPENSHELL_SANDBOX_NAMESPACE = process.env.OPENSHELL_SANDBOX_NAMESPACE || "openshell"
|
|
||||||
|
|
||||||
function sleep(ms: number) {
|
function sleep(ms: number) {
|
||||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||||
@@ -18,24 +17,25 @@ function validateSandboxName(value: string) {
|
|||||||
return value
|
return value
|
||||||
}
|
}
|
||||||
|
|
||||||
async function runKubectl(args: string[]) {
|
async function runOpenShell(args: string[], timeout = 30000) {
|
||||||
return await new Promise<{ stdout: string; stderr: string; code: number | null }>((resolve, reject) => {
|
const { stdout, stderr } = await execFileAsync(OPENSHELL_BIN, args, {
|
||||||
const child = spawn(DOCKER_BIN, ["exec", OPENSHELL_CLUSTER_CONTAINER, "kubectl", ...args], {
|
env: hostCommandEnv({
|
||||||
stdio: ["ignore", "pipe", "pipe"],
|
OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY || "nemoclaw",
|
||||||
})
|
}),
|
||||||
let stdout = ""
|
timeout,
|
||||||
let stderr = ""
|
maxBuffer: 4 * 1024 * 1024,
|
||||||
child.stdout.on("data", (chunk) => { stdout += String(chunk) })
|
|
||||||
child.stderr.on("data", (chunk) => { stderr += String(chunk) })
|
|
||||||
child.on("error", reject)
|
|
||||||
child.on("close", (code) => resolve({ stdout: stdout.trim(), stderr: stderr.trim(), code }))
|
|
||||||
})
|
})
|
||||||
|
return { stdout: String(stdout).trim(), stderr: String(stderr).trim() }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runSandboxShell(sandboxName: string, script: string, timeout = 30000) {
|
||||||
|
return runOpenShell(["sandbox", "exec", "-n", sandboxName, "--", "sh", "-lc", script], timeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
async function waitForSandboxReady(sandboxName: string, timeoutMs: number, intervalMs: number) {
|
async function waitForSandboxReady(sandboxName: string, timeoutMs: number, intervalMs: number) {
|
||||||
const startedAt = Date.now()
|
const startedAt = Date.now()
|
||||||
let attempts = 0
|
let attempts = 0
|
||||||
let lastError: string | null = null
|
let lastError = ""
|
||||||
|
|
||||||
while (Date.now() - startedAt < timeoutMs) {
|
while (Date.now() - startedAt < timeoutMs) {
|
||||||
attempts += 1
|
attempts += 1
|
||||||
@@ -54,46 +54,67 @@ async function waitForSandboxReady(sandboxName: string, timeoutMs: number, inter
|
|||||||
return { ready: false, attempts, elapsedMs: Date.now() - startedAt, lastError }
|
return { ready: false, attempts, elapsedMs: Date.now() - startedAt, lastError }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function restartOpenClawGatewayScript() {
|
||||||
|
return [
|
||||||
|
`port=${SANDBOX_DASHBOARD_REMOTE_PORT}`,
|
||||||
|
"for p in /proc/[0-9]*; do",
|
||||||
|
" cmd=$(tr '\\0' ' ' < \"$p/cmdline\" 2>/dev/null || true)",
|
||||||
|
" case \"$cmd\" in",
|
||||||
|
" *'openclaw gateway run'*) kill \"${p##*/}\" 2>/dev/null || true ;;",
|
||||||
|
" esac",
|
||||||
|
"done",
|
||||||
|
"sleep 1",
|
||||||
|
"if command -v openclaw >/dev/null 2>&1; then openclaw_bin=$(command -v openclaw);",
|
||||||
|
"elif [ -x /usr/local/bin/openclaw ]; then openclaw_bin=/usr/local/bin/openclaw;",
|
||||||
|
"else echo 'openclaw command not found in sandbox' >&2; exit 127; fi",
|
||||||
|
"nohup \"$openclaw_bin\" gateway run --allow-unconfigured --bind loopback --port \"$port\" >/tmp/gateway.log 2>&1 &",
|
||||||
|
"for i in 1 2 3 4 5 6 7 8 9 10; do",
|
||||||
|
" curl -fsS --max-time 2 \"http://127.0.0.1:$port/\" >/dev/null 2>&1 && exit 0",
|
||||||
|
" sleep 1",
|
||||||
|
"done",
|
||||||
|
"echo 'OpenClaw gateway did not answer after restart. Last log lines:' >&2",
|
||||||
|
"tail -40 /tmp/gateway.log >&2 2>/dev/null || true",
|
||||||
|
"exit 1",
|
||||||
|
].join("\n")
|
||||||
|
}
|
||||||
|
|
||||||
export async function POST(
|
export async function POST(
|
||||||
_request: Request,
|
_request: Request,
|
||||||
{ params }: { params: Promise<{ sandboxId: string }> }
|
{ params }: { params: Promise<{ sandboxId: string }> },
|
||||||
) {
|
) {
|
||||||
const startedAt = Date.now()
|
const startedAt = Date.now()
|
||||||
try {
|
try {
|
||||||
const { sandboxId } = await params
|
const { sandboxId } = await params
|
||||||
const resolved = await resolveSandboxRef(sandboxId)
|
const resolved = await resolveSandboxRef(sandboxId)
|
||||||
const sandboxName = validateSandboxName(resolved.name)
|
const sandboxName = validateSandboxName(resolved.name)
|
||||||
const deleted = await runKubectl(["delete", "pod", sandboxName, "-n", OPENSHELL_SANDBOX_NAMESPACE, "--wait=false"])
|
const readiness = await waitForSandboxReady(sandboxName, 15000, 1000)
|
||||||
if (deleted.code !== 0 && !/not found/i.test(`${deleted.stdout}\n${deleted.stderr}`)) {
|
if (!readiness.ready) {
|
||||||
throw new Error(deleted.stderr || deleted.stdout || "Failed to restart sandbox pod")
|
|
||||||
}
|
|
||||||
|
|
||||||
const readiness = await waitForSandboxReady(sandboxName, 90000, 2000)
|
|
||||||
let inferenceApply = null
|
|
||||||
if (readiness.ready) {
|
|
||||||
const inferenceConfig = await getSandboxInferenceConfig(resolved.id)
|
|
||||||
if (inferenceConfig.routes.some((route) => route.enabled)) {
|
|
||||||
inferenceApply = await applySandboxInferenceProfile(resolved.id, sandboxName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
ok: readiness.ready,
|
ok: false,
|
||||||
restarted: readiness.ready,
|
restarted: false,
|
||||||
sandboxId: resolved.id,
|
sandboxId: resolved.id,
|
||||||
sandboxName,
|
sandboxName,
|
||||||
deletion: deleted,
|
|
||||||
readiness,
|
readiness,
|
||||||
inferenceApply,
|
|
||||||
elapsedMs: Date.now() - startedAt,
|
elapsedMs: Date.now() - startedAt,
|
||||||
note: readiness.ready
|
note: "Sandbox was not Ready, so the dashboard runtime was not restarted.",
|
||||||
? inferenceApply
|
}, { status: 409 })
|
||||||
? "Sandbox pod restarted, OpenShell reports it Ready, and the saved inference profile was reapplied."
|
}
|
||||||
: "Sandbox pod restarted and OpenShell reports it Ready."
|
|
||||||
: "Restart was requested, but OpenShell has not reported the sandbox Ready yet.",
|
const runtime = await runSandboxShell(sandboxName, restartOpenClawGatewayScript(), 45000)
|
||||||
}, { status: readiness.ready ? 200 : 202 })
|
|
||||||
|
return NextResponse.json({
|
||||||
|
ok: true,
|
||||||
|
restarted: true,
|
||||||
|
restartMode: "openclaw-runtime",
|
||||||
|
sandboxId: resolved.id,
|
||||||
|
sandboxName,
|
||||||
|
readiness,
|
||||||
|
runtime,
|
||||||
|
elapsedMs: Date.now() - startedAt,
|
||||||
|
note: "OpenClaw runtime restarted inside the sandbox. The sandbox pod was not deleted.",
|
||||||
|
})
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const message = error instanceof Error ? error.message : "Failed to restart sandbox"
|
const message = error instanceof Error ? error.message : "Failed to restart sandbox runtime"
|
||||||
return NextResponse.json({ ok: false, restarted: false, error: message }, { status: /required|invalid/.test(message) ? 400 : 500 })
|
return NextResponse.json({ ok: false, restarted: false, error: message }, { status: /required|invalid/.test(message) ? 400 : 500 })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -213,20 +213,20 @@ export default function SandboxList({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const restartSandbox = async () => {
|
const restartSandboxRuntime = async () => {
|
||||||
if (!selectedSandbox || restartInProgress) return
|
if (!selectedSandbox || restartInProgress) return
|
||||||
try {
|
try {
|
||||||
setRestartInProgress(true)
|
setRestartInProgress(true)
|
||||||
setDashboardMessage(`Restarting sandbox ${selectedSandbox.name}...`)
|
setDashboardMessage(`Restarting OpenClaw runtime for ${selectedSandbox.name}...`)
|
||||||
const response = await fetch(`/api/sandbox/${encodeURIComponent(selectedSandbox.id)}/restart`, {
|
const response = await fetch(`/api/sandbox/${encodeURIComponent(selectedSandbox.id)}/restart`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
})
|
})
|
||||||
const data = await response.json()
|
const data = await response.json()
|
||||||
if (!response.ok) throw new Error(data.error || data.note || 'Failed to restart sandbox')
|
if (!response.ok) throw new Error(data.error || data.note || 'Failed to restart OpenClaw runtime')
|
||||||
setDashboardMessage(data.note || `Sandbox ${selectedSandbox.name} restarted.`)
|
setDashboardMessage(data.note || `OpenClaw runtime restarted for ${selectedSandbox.name}.`)
|
||||||
await onInventoryRefresh()
|
await onInventoryRefresh()
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
setDashboardMessage(error instanceof Error ? error.message : 'Failed to restart sandbox.')
|
setDashboardMessage(error instanceof Error ? error.message : 'Failed to restart OpenClaw runtime.')
|
||||||
} finally {
|
} finally {
|
||||||
setRestartInProgress(false)
|
setRestartInProgress(false)
|
||||||
}
|
}
|
||||||
@@ -538,11 +538,11 @@ export default function SandboxList({
|
|||||||
Start OpenClaw Gateway Dashboard
|
Start OpenClaw Gateway Dashboard
|
||||||
</button>
|
</button>
|
||||||
<button
|
<button
|
||||||
onClick={restartSandbox}
|
onClick={restartSandboxRuntime}
|
||||||
disabled={restartInProgress}
|
disabled={restartInProgress}
|
||||||
className="action-button px-3 py-2"
|
className="action-button px-3 py-2"
|
||||||
>
|
>
|
||||||
{restartInProgress ? 'Restarting Sandbox...' : 'Restart Sandbox'}
|
{restartInProgress ? 'Restarting Runtime...' : 'Restart OpenClaw Runtime'}
|
||||||
</button>
|
</button>
|
||||||
<button
|
<button
|
||||||
onClick={async () => {
|
onClick={async () => {
|
||||||
|
|||||||
@@ -105,9 +105,10 @@ assert.match(sandboxInferencePanelSource, /Add Route/, 'selected sandbox UI must
|
|||||||
assert.match(sandboxInferenceApplyLibSource, /\/sandbox\/\.openclaw\/openclaw\.json/, 'live apply must patch OpenClaw config inside the running sandbox')
|
assert.match(sandboxInferenceApplyLibSource, /\/sandbox\/\.openclaw\/openclaw\.json/, 'live apply must patch OpenClaw config inside the running sandbox')
|
||||||
assert.match(sandboxInferenceApplyLibSource, /sha256sum \/sandbox\/\.openclaw\/openclaw\.json > \/sandbox\/\.openclaw\/\.config-hash/, 'live apply must refresh the immutable OpenClaw config hash')
|
assert.match(sandboxInferenceApplyLibSource, /sha256sum \/sandbox\/\.openclaw\/openclaw\.json > \/sandbox\/\.openclaw\/\.config-hash/, 'live apply must refresh the immutable OpenClaw config hash')
|
||||||
assert.match(sandboxInferenceApplyLibSource, /\["inference", "set", "--no-verify", "--provider", primary\.provider, "--model", primary\.model\]/, 'live apply must point OpenShell inference at the primary route')
|
assert.match(sandboxInferenceApplyLibSource, /\["inference", "set", "--no-verify", "--provider", primary\.provider, "--model", primary\.model\]/, 'live apply must point OpenShell inference at the primary route')
|
||||||
assert.match(sandboxRestartRouteSource, /\["delete", "pod", sandboxName, "-n", OPENSHELL_SANDBOX_NAMESPACE, "--wait=false"\]/, 'restart route must restart the backing sandbox pod')
|
assert.doesNotMatch(sandboxRestartRouteSource, /delete", "pod"|kubectl delete/, 'restart route must not delete the backing sandbox pod')
|
||||||
assert.match(sandboxRestartRouteSource, /waitForSandboxReady\(sandboxName, 90000, 2000\)/, 'restart route must wait for OpenShell to report the sandbox ready')
|
assert.match(sandboxRestartRouteSource, /restartMode: "openclaw-runtime"/, 'restart route must restart only the in-sandbox OpenClaw runtime')
|
||||||
assert.match(sandboxListSource, /Restart Sandbox/, 'sandbox details must expose a restart sandbox button')
|
assert.match(sandboxRestartRouteSource, /waitForSandboxReady\(sandboxName, 15000, 1000\)/, 'restart route must verify OpenShell reports the sandbox ready before runtime restart')
|
||||||
|
assert.match(sandboxListSource, /Restart OpenClaw Runtime/, 'sandbox details must expose a scoped runtime restart button')
|
||||||
assert.match(sandboxListSource, /\/api\/sandbox\/\$\{encodeURIComponent\(selectedSandbox\.id\)\}\/restart/, 'restart button must call the sandbox restart endpoint')
|
assert.match(sandboxListSource, /\/api\/sandbox\/\$\{encodeURIComponent\(selectedSandbox\.id\)\}\/restart/, 'restart button must call the sandbox restart endpoint')
|
||||||
assert.doesNotMatch(sandboxListSource, /setInterval\(loadPermissionFeeds/, 'sandbox list must not poll every sandbox permission feed')
|
assert.doesNotMatch(sandboxListSource, /setInterval\(loadPermissionFeeds/, 'sandbox list must not poll every sandbox permission feed')
|
||||||
assert.match(sandboxListSource, /loadPermissionFeed\(selectedSandbox\)/, 'policy drawer must load the selected sandbox permission feed on demand')
|
assert.match(sandboxListSource, /loadPermissionFeed\(selectedSandbox\)/, 'policy drawer must load the selected sandbox permission feed on demand')
|
||||||
|
|||||||
Reference in New Issue
Block a user