Add sandbox restart control
This commit is contained in:
@@ -0,0 +1,86 @@
|
|||||||
|
import { NextResponse } from "next/server"
|
||||||
|
import { spawn } from "node:child_process"
|
||||||
|
import { inspectSandbox, resolveSandboxRef } from "@/app/lib/openshellHost"
|
||||||
|
|
||||||
|
const DOCKER_BIN = process.env.DOCKER_BIN || "docker"
|
||||||
|
const OPENSHELL_CLUSTER_CONTAINER = process.env.OPENSHELL_CLUSTER_CONTAINER || "openshell-cluster-nemoclaw"
|
||||||
|
const OPENSHELL_SANDBOX_NAMESPACE = process.env.OPENSHELL_SANDBOX_NAMESPACE || "openshell"
|
||||||
|
|
||||||
|
function sleep(ms: number) {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateSandboxName(value: string) {
|
||||||
|
if (!value || value.length > 63) throw new Error("sandbox name is required")
|
||||||
|
if (!/^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/.test(value)) throw new Error("invalid sandbox name")
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runKubectl(args: string[]) {
|
||||||
|
return await new Promise<{ stdout: string; stderr: string; code: number | null }>((resolve, reject) => {
|
||||||
|
const child = spawn(DOCKER_BIN, ["exec", OPENSHELL_CLUSTER_CONTAINER, "kubectl", ...args], {
|
||||||
|
stdio: ["ignore", "pipe", "pipe"],
|
||||||
|
})
|
||||||
|
let stdout = ""
|
||||||
|
let stderr = ""
|
||||||
|
child.stdout.on("data", (chunk) => { stdout += String(chunk) })
|
||||||
|
child.stderr.on("data", (chunk) => { stderr += String(chunk) })
|
||||||
|
child.on("error", reject)
|
||||||
|
child.on("close", (code) => resolve({ stdout: stdout.trim(), stderr: stderr.trim(), code }))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForSandboxReady(sandboxName: string, timeoutMs: number, intervalMs: number) {
|
||||||
|
const startedAt = Date.now()
|
||||||
|
let attempts = 0
|
||||||
|
let lastError: string | null = null
|
||||||
|
|
||||||
|
while (Date.now() - startedAt < timeoutMs) {
|
||||||
|
attempts += 1
|
||||||
|
try {
|
||||||
|
const inspection = await inspectSandbox(sandboxName)
|
||||||
|
if (inspection.rawPhase === "Ready") {
|
||||||
|
return { ready: true, attempts, elapsedMs: Date.now() - startedAt, inspection }
|
||||||
|
}
|
||||||
|
lastError = `sandbox phase ${inspection.rawPhase || inspection.phase || "unknown"}`
|
||||||
|
} catch (error) {
|
||||||
|
lastError = error instanceof Error ? error.message : String(error ?? "Sandbox lookup failed")
|
||||||
|
}
|
||||||
|
await sleep(intervalMs)
|
||||||
|
}
|
||||||
|
|
||||||
|
return { ready: false, attempts, elapsedMs: Date.now() - startedAt, lastError }
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function POST(
|
||||||
|
_request: Request,
|
||||||
|
{ params }: { params: Promise<{ sandboxId: string }> }
|
||||||
|
) {
|
||||||
|
const startedAt = Date.now()
|
||||||
|
try {
|
||||||
|
const { sandboxId } = await params
|
||||||
|
const resolved = await resolveSandboxRef(sandboxId)
|
||||||
|
const sandboxName = validateSandboxName(resolved.name)
|
||||||
|
const deleted = await runKubectl(["delete", "pod", sandboxName, "-n", OPENSHELL_SANDBOX_NAMESPACE, "--wait=false"])
|
||||||
|
if (deleted.code !== 0 && !/not found/i.test(`${deleted.stdout}\n${deleted.stderr}`)) {
|
||||||
|
throw new Error(deleted.stderr || deleted.stdout || "Failed to restart sandbox pod")
|
||||||
|
}
|
||||||
|
|
||||||
|
const readiness = await waitForSandboxReady(sandboxName, 90000, 2000)
|
||||||
|
return NextResponse.json({
|
||||||
|
ok: readiness.ready,
|
||||||
|
restarted: readiness.ready,
|
||||||
|
sandboxId: resolved.id,
|
||||||
|
sandboxName,
|
||||||
|
deletion: deleted,
|
||||||
|
readiness,
|
||||||
|
elapsedMs: Date.now() - startedAt,
|
||||||
|
note: readiness.ready
|
||||||
|
? "Sandbox pod restarted and OpenShell reports it Ready."
|
||||||
|
: "Restart was requested, but OpenShell has not reported the sandbox Ready yet.",
|
||||||
|
}, { status: readiness.ready ? 200 : 202 })
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : "Failed to restart sandbox"
|
||||||
|
return NextResponse.json({ ok: false, restarted: false, error: message }, { status: /required|invalid/.test(message) ? 400 : 500 })
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -51,6 +51,7 @@ export default function SandboxList({
|
|||||||
}: SandboxListProps) {
|
}: SandboxListProps) {
|
||||||
const [terminalMessage, setTerminalMessage] = useState<string>('')
|
const [terminalMessage, setTerminalMessage] = useState<string>('')
|
||||||
const [dashboardMessage, setDashboardMessage] = useState<string>('')
|
const [dashboardMessage, setDashboardMessage] = useState<string>('')
|
||||||
|
const [restartInProgress, setRestartInProgress] = useState(false)
|
||||||
const [telemetry, setTelemetry] = useState<TelemetryData>({
|
const [telemetry, setTelemetry] = useState<TelemetryData>({
|
||||||
cpu: 0, memory: 0, disk: 0, timestamp: new Date().toISOString()
|
cpu: 0, memory: 0, disk: 0, timestamp: new Date().toISOString()
|
||||||
})
|
})
|
||||||
@@ -73,6 +74,25 @@ export default function SandboxList({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const restartSandbox = async () => {
|
||||||
|
if (!selectedSandbox || restartInProgress) return
|
||||||
|
try {
|
||||||
|
setRestartInProgress(true)
|
||||||
|
setDashboardMessage(`Restarting sandbox ${selectedSandbox.name}...`)
|
||||||
|
const response = await fetch(`/api/sandbox/${encodeURIComponent(selectedSandbox.id)}/restart`, {
|
||||||
|
method: 'POST',
|
||||||
|
})
|
||||||
|
const data = await response.json()
|
||||||
|
if (!response.ok) throw new Error(data.error || data.note || 'Failed to restart sandbox')
|
||||||
|
setDashboardMessage(data.note || `Sandbox ${selectedSandbox.name} restarted.`)
|
||||||
|
await onInventoryRefresh()
|
||||||
|
} catch (error) {
|
||||||
|
setDashboardMessage(error instanceof Error ? error.message : 'Failed to restart sandbox.')
|
||||||
|
} finally {
|
||||||
|
setRestartInProgress(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="space-y-6">
|
<div className="space-y-6">
|
||||||
{sandboxes.length === 0 ? (
|
{sandboxes.length === 0 ? (
|
||||||
@@ -188,6 +208,13 @@ export default function SandboxList({
|
|||||||
>
|
>
|
||||||
Start OpenClaw Gateway Dashboard
|
Start OpenClaw Gateway Dashboard
|
||||||
</button>
|
</button>
|
||||||
|
<button
|
||||||
|
onClick={restartSandbox}
|
||||||
|
disabled={restartInProgress}
|
||||||
|
className="px-3 py-2 rounded-sm bg-[var(--background-tertiary)] text-[var(--foreground)] text-xs font-mono uppercase tracking-wider hover:border-[var(--nvidia-green)] border border-[var(--border-subtle)] disabled:opacity-50"
|
||||||
|
>
|
||||||
|
{restartInProgress ? 'Restarting Sandbox...' : 'Restart Sandbox'}
|
||||||
|
</button>
|
||||||
<button
|
<button
|
||||||
onClick={() => {
|
onClick={() => {
|
||||||
const nextUrl = buildOperatorTerminalRoute({
|
const nextUrl = buildOperatorTerminalRoute({
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ const inferenceRoutePath = path.join(root, 'app/api/inference/route.ts')
|
|||||||
const ollamaModelsRoutePath = path.join(root, 'app/api/ollama/models/route.ts')
|
const ollamaModelsRoutePath = path.join(root, 'app/api/ollama/models/route.ts')
|
||||||
const sandboxInferenceRoutePath = path.join(root, 'app/api/sandbox/[sandboxId]/inference/route.ts')
|
const sandboxInferenceRoutePath = path.join(root, 'app/api/sandbox/[sandboxId]/inference/route.ts')
|
||||||
const sandboxInferenceApplyRoutePath = path.join(root, 'app/api/sandbox/[sandboxId]/inference/apply/route.ts')
|
const sandboxInferenceApplyRoutePath = path.join(root, 'app/api/sandbox/[sandboxId]/inference/apply/route.ts')
|
||||||
|
const sandboxRestartRoutePath = path.join(root, 'app/api/sandbox/[sandboxId]/restart/route.ts')
|
||||||
const hookPath = path.join(root, 'app/hooks/useSandboxInventory.ts')
|
const hookPath = path.join(root, 'app/hooks/useSandboxInventory.ts')
|
||||||
const pagePath = path.join(root, 'app/page.tsx')
|
const pagePath = path.join(root, 'app/page.tsx')
|
||||||
const configPanelPath = path.join(root, 'app/components/ConfigurationPanel.tsx')
|
const configPanelPath = path.join(root, 'app/components/ConfigurationPanel.tsx')
|
||||||
@@ -25,6 +26,7 @@ const inferenceRouteSource = await readFile(inferenceRoutePath, 'utf8')
|
|||||||
const ollamaModelsRouteSource = await readFile(ollamaModelsRoutePath, 'utf8')
|
const ollamaModelsRouteSource = await readFile(ollamaModelsRoutePath, 'utf8')
|
||||||
const sandboxInferenceRouteSource = await readFile(sandboxInferenceRoutePath, 'utf8')
|
const sandboxInferenceRouteSource = await readFile(sandboxInferenceRoutePath, 'utf8')
|
||||||
const sandboxInferenceApplyRouteSource = await readFile(sandboxInferenceApplyRoutePath, 'utf8')
|
const sandboxInferenceApplyRouteSource = await readFile(sandboxInferenceApplyRoutePath, 'utf8')
|
||||||
|
const sandboxRestartRouteSource = await readFile(sandboxRestartRoutePath, 'utf8')
|
||||||
const hookSource = await readFile(hookPath, 'utf8')
|
const hookSource = await readFile(hookPath, 'utf8')
|
||||||
const pageSource = await readFile(pagePath, 'utf8')
|
const pageSource = await readFile(pagePath, 'utf8')
|
||||||
const configPanelSource = await readFile(configPanelPath, 'utf8')
|
const configPanelSource = await readFile(configPanelPath, 'utf8')
|
||||||
@@ -82,6 +84,10 @@ assert.match(sandboxInferencePanelSource, /Add Route/, 'selected sandbox UI must
|
|||||||
assert.match(sandboxInferenceApplyRouteSource, /\/sandbox\/\.openclaw\/openclaw\.json/, 'live apply must patch OpenClaw config inside the running sandbox')
|
assert.match(sandboxInferenceApplyRouteSource, /\/sandbox\/\.openclaw\/openclaw\.json/, 'live apply must patch OpenClaw config inside the running sandbox')
|
||||||
assert.match(sandboxInferenceApplyRouteSource, /sha256sum \/sandbox\/\.openclaw\/openclaw\.json > \/sandbox\/\.openclaw\/\.config-hash/, 'live apply must refresh the immutable OpenClaw config hash')
|
assert.match(sandboxInferenceApplyRouteSource, /sha256sum \/sandbox\/\.openclaw\/openclaw\.json > \/sandbox\/\.openclaw\/\.config-hash/, 'live apply must refresh the immutable OpenClaw config hash')
|
||||||
assert.match(sandboxInferenceApplyRouteSource, /\["inference", "set", "--no-verify", "--provider", primary\.provider, "--model", primary\.model\]/, 'live apply must point OpenShell inference at the primary route')
|
assert.match(sandboxInferenceApplyRouteSource, /\["inference", "set", "--no-verify", "--provider", primary\.provider, "--model", primary\.model\]/, 'live apply must point OpenShell inference at the primary route')
|
||||||
|
assert.match(sandboxRestartRouteSource, /\["delete", "pod", sandboxName, "-n", OPENSHELL_SANDBOX_NAMESPACE, "--wait=false"\]/, 'restart route must restart the backing sandbox pod')
|
||||||
|
assert.match(sandboxRestartRouteSource, /waitForSandboxReady\(sandboxName, 90000, 2000\)/, 'restart route must wait for OpenShell to report the sandbox ready')
|
||||||
|
assert.match(sandboxListSource, /Restart Sandbox/, 'sandbox details must expose a restart sandbox button')
|
||||||
|
assert.match(sandboxListSource, /\/api\/sandbox\/\$\{encodeURIComponent\(selectedSandbox\.id\)\}\/restart/, 'restart button must call the sandbox restart endpoint')
|
||||||
assert.match(sandboxListSource, /<SandboxInferencePanel sandbox=\{selectedSandbox\} \/>/, 'sandbox list must render the per-sandbox inference profile panel for the selected sandbox')
|
assert.match(sandboxListSource, /<SandboxInferencePanel sandbox=\{selectedSandbox\} \/>/, 'sandbox list must render the per-sandbox inference profile panel for the selected sandbox')
|
||||||
assert.match(sidebarSource, /Inference Endpoints/, 'sidebar settings item must be named Inference Endpoints')
|
assert.match(sidebarSource, /Inference Endpoints/, 'sidebar settings item must be named Inference Endpoints')
|
||||||
assert.match(pageSource, /<InferenceEndpointPanel \/>/, 'settings view must render inference endpoint panel')
|
assert.match(pageSource, /<InferenceEndpointPanel \/>/, 'settings view must render inference endpoint panel')
|
||||||
|
|||||||
Reference in New Issue
Block a user