fix: follow native agent gateway lifecycle (#49)

Co-authored-by: mmckeen-nv <mmckeen-nv@users.noreply.github.com>
This commit is contained in:
mmckeen-nv
2026-09-17 23:10:53 -07:00
committed by GitHub
co-authored by mmckeen-nv
parent 85c6777c55
commit 4e2d51e84a
5 changed files with 31 additions and 78 deletions
+2
View File
@@ -225,6 +225,8 @@ The dashboard shells out to the OpenShell CLI for several operations:
The **Quick Deploy New NemoClaw Sandbox** path clones a registered image through OpenShell, but it deliberately does not write the clone into NemoClaw's private `sandboxes.json`. Current NemoClaw entries include lifecycle identity, workload provenance, and agent-specific state that must be issued by NemoClaw itself. The clone remains visible and operable through OpenShell Control; use **Fresh NemoClaw Image** when the new sandbox must participate in NemoClaw-managed lifecycle operations. The **Quick Deploy New NemoClaw Sandbox** path clones a registered image through OpenShell, but it deliberately does not write the clone into NemoClaw's private `sandboxes.json`. Current NemoClaw entries include lifecycle identity, workload provenance, and agent-specific state that must be issued by NemoClaw itself. The clone remains visible and operable through OpenShell Control; use **Fresh NemoClaw Image** when the new sandbox must participate in NemoClaw-managed lifecycle operations.
Sandbox gateway restarts use NemoClaw's native-agent lifecycle command. The controller does not kill or launch OpenClaw or Hermes gateway processes directly; if NemoClaw cannot verify the restart, the request fails without bypassing the agent-owned lifecycle boundary.
OpenClaw dashboard access is loopback-only inside the host/sandbox context, so the UI uses local proxy routes: OpenClaw dashboard access is loopback-only inside the host/sandbox context, so the UI uses local proxy routes:
- `/api/openshell/dashboard/proxy` - `/api/openshell/dashboard/proxy`
+14 -60
View File
@@ -1,13 +1,7 @@
import { NextResponse } from "next/server" import { NextResponse } from "next/server"
import { execFile } from "node:child_process" import { restartSandboxGatewayWithNemoClaw } from "@/app/lib/nemoclawCli"
import { promisify } from "node:util"
import { OPENSHELL_BIN, hostCommandEnv } from "@/app/lib/hostCommands"
import { recoverSandboxWithNemoClaw } from "@/app/lib/nemoclawCli"
import { inspectSandbox, resolveSandboxRef } from "@/app/lib/openshellHost" import { inspectSandbox, resolveSandboxRef } from "@/app/lib/openshellHost"
const execFileAsync = promisify(execFile)
const SANDBOX_DASHBOARD_REMOTE_PORT = Number.parseInt(process.env.OPENCLAW_SANDBOX_DASHBOARD_REMOTE_PORT || "18789", 10)
function sleep(ms: number) { function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms)) return new Promise((resolve) => setTimeout(resolve, ms))
} }
@@ -18,20 +12,6 @@ function validateSandboxName(value: string) {
return value return value
} }
async function runOpenShell(args: string[], timeout = 30000) {
const { stdout, stderr } = await execFileAsync(OPENSHELL_BIN, args, {
env: hostCommandEnv({
OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY?.trim() || undefined,
}),
timeout,
maxBuffer: 4 * 1024 * 1024,
})
return { stdout: String(stdout).trim(), stderr: String(stderr).trim() }
}
async function runSandboxShell(sandboxName: string, script: string, timeout = 30000) {
return runOpenShell(["sandbox", "exec", "-n", sandboxName, "--", "sh", "-lc", script], timeout)
}
async function waitForSandboxReady(sandboxName: string, timeoutMs: number, intervalMs: number) { async function waitForSandboxReady(sandboxName: string, timeoutMs: number, intervalMs: number) {
const startedAt = Date.now() const startedAt = Date.now()
@@ -55,29 +35,6 @@ async function waitForSandboxReady(sandboxName: string, timeoutMs: number, inter
return { ready: false, attempts, elapsedMs: Date.now() - startedAt, lastError } return { ready: false, attempts, elapsedMs: Date.now() - startedAt, lastError }
} }
function restartOpenClawGatewayScript() {
return [
`port=${SANDBOX_DASHBOARD_REMOTE_PORT}`,
"for p in /proc/[0-9]*; do",
" cmd=$(tr '\\0' ' ' < \"$p/cmdline\" 2>/dev/null || true)",
" case \"$cmd\" in",
" *'openclaw gateway run'*) kill \"${p##*/}\" 2>/dev/null || true ;;",
" esac",
"done",
"sleep 1",
"if command -v openclaw >/dev/null 2>&1; then openclaw_bin=$(command -v openclaw);",
"elif [ -x /usr/local/bin/openclaw ]; then openclaw_bin=/usr/local/bin/openclaw;",
"else echo 'openclaw command not found in sandbox' >&2; exit 127; fi",
"nohup \"$openclaw_bin\" gateway run --allow-unconfigured --bind loopback --port \"$port\" >/tmp/gateway.log 2>&1 &",
"for i in 1 2 3 4 5 6 7 8 9 10; do",
" curl -fsS --max-time 2 \"http://127.0.0.1:$port/\" >/dev/null 2>&1 && exit 0",
" sleep 1",
"done",
"echo 'OpenClaw gateway did not answer after restart. Last log lines:' >&2",
"tail -40 /tmp/gateway.log >&2 2>/dev/null || true",
"exit 1",
].join("\n")
}
export async function POST( export async function POST(
_request: Request, _request: Request,
@@ -101,37 +58,34 @@ export async function POST(
}, { status: 409 }) }, { status: 409 })
} }
const nemoclawRecover = await recoverSandboxWithNemoClaw(sandboxName) const nemoclawRestart = await restartSandboxGatewayWithNemoClaw(sandboxName)
if (nemoclawRecover.attempted && nemoclawRecover.ok) { if (nemoclawRestart.attempted && nemoclawRestart.ok) {
return NextResponse.json({ return NextResponse.json({
ok: true, ok: true,
restarted: true, restarted: true,
restartMode: "nemoclaw-recover", restartMode: "nemoclaw-native-gateway",
sandboxId: resolved.id, sandboxId: resolved.id,
sandboxName, sandboxName,
readiness, readiness,
nemoclawRecover, nemoclawRestart,
elapsedMs: Date.now() - startedAt, elapsedMs: Date.now() - startedAt,
note: "NemoClaw recover completed. The sandbox runtime and dashboard forward were checked without deleting the sandbox.", note: "NemoClaw asked the sandbox's native agent to restart its gateway, then verified gateway health and host forwards.",
}) })
} }
const runtime = await runSandboxShell(sandboxName, restartOpenClawGatewayScript(), 45000)
return NextResponse.json({ return NextResponse.json({
ok: true, ok: false,
restarted: true, restarted: false,
restartMode: "openclaw-runtime", restartMode: "nemoclaw-native-gateway",
sandboxId: resolved.id, sandboxId: resolved.id,
sandboxName, sandboxName,
readiness, readiness,
nemoclawRecover, nemoclawRestart,
runtime,
elapsedMs: Date.now() - startedAt, elapsedMs: Date.now() - startedAt,
note: nemoclawRecover.attempted note: nemoclawRestart.attempted
? "NemoClaw recover did not complete, so the dashboard fell back to restarting the in-sandbox OpenClaw runtime. The sandbox pod was not deleted." ? "NemoClaw could not complete the native gateway restart. The controller did not bypass the agent-owned lifecycle boundary."
: "OpenClaw runtime restarted inside the sandbox. The sandbox pod was not deleted.", : "The installed NemoClaw CLI does not support the native gateway restart command. Upgrade NemoClaw before retrying.",
}) }, { status: nemoclawRestart.attempted ? 502 : 409 })
} catch (error) { } catch (error) {
const message = error instanceof Error ? error.message : "Failed to restart sandbox runtime" const message = error instanceof Error ? error.message : "Failed to restart sandbox runtime"
return NextResponse.json({ ok: false, restarted: false, error: message }, { status: /required|invalid/.test(message) ? 400 : 500 }) return NextResponse.json({ ok: false, restarted: false, error: message }, { status: /required|invalid/.test(message) ? 400 : 500 })
+6 -6
View File
@@ -3,7 +3,7 @@ import { promisify } from "node:util"
import { NEMOCLAW_BIN, NODE_BIN, commandExists, hostCommandEnv } from "./hostCommands" import { NEMOCLAW_BIN, NODE_BIN, commandExists, hostCommandEnv } from "./hostCommands"
const execFileAsync = promisify(execFile) const execFileAsync = promisify(execFile)
const NEMOCLAW_RECOVERY_MIN_VERSION = "0.0.37" const NEMOCLAW_LIFECYCLE_MIN_VERSION = "0.0.37"
type NemoClawInvocation = { type NemoClawInvocation = {
file: string file: string
@@ -143,8 +143,8 @@ async function hasModernNemoClawSurface() {
const version = await getNemoClawVersion() const version = await getNemoClawVersion()
return { return {
...version, ...version,
supported: version.version ? versionGte(version.version, NEMOCLAW_RECOVERY_MIN_VERSION) : false, supported: version.version ? versionGte(version.version, NEMOCLAW_LIFECYCLE_MIN_VERSION) : false,
minimumVersion: NEMOCLAW_RECOVERY_MIN_VERSION, minimumVersion: NEMOCLAW_LIFECYCLE_MIN_VERSION,
} }
} }
@@ -182,7 +182,7 @@ export async function getNemoClawDoctorReport(sandboxName: string) {
} }
} }
export async function recoverSandboxWithNemoClaw(sandboxName: string) { export async function restartSandboxGatewayWithNemoClaw(sandboxName: string) {
const capability = await hasModernNemoClawSurface() const capability = await hasModernNemoClawSurface()
if (!capability.supported) { if (!capability.supported) {
return { return {
@@ -196,12 +196,12 @@ export async function recoverSandboxWithNemoClaw(sandboxName: string) {
stderr: "", stderr: "",
exitCode: null as number | null, exitCode: null as number | null,
error: capability.available error: capability.available
? `NemoClaw ${capability.version || "unknown"} does not expose sandbox recover` ? `NemoClaw ${capability.version || "unknown"} does not expose sandbox gateway restart`
: capability.error || "NemoClaw CLI is not available", : capability.error || "NemoClaw CLI is not available",
} }
} }
const result = await runNemoClaw([sandboxName, "recover"], 90000) const result = await runNemoClaw(["sandbox", "gateway", "restart", sandboxName], 90000)
return { return {
available: true, available: true,
supported: true, supported: true,
+7 -11
View File
@@ -13,14 +13,14 @@ const [helperSource, healthRouteSource, restartRouteSource] = await Promise.all(
readFile(restartRoutePath, 'utf8'), readFile(restartRoutePath, 'utf8'),
]) ])
assert.match(helperSource, /const NEMOCLAW_RECOVERY_MIN_VERSION = "0\.0\.37"/, 'NemoClaw doctor/recover integration must be gated behind the v0.0.37 CLI surface') assert.match(helperSource, /const NEMOCLAW_LIFECYCLE_MIN_VERSION = "0\.0\.37"/, 'NemoClaw doctor/lifecycle integration must be gated behind the v0.0.37 CLI surface')
assert.match(helperSource, /NEMOCLAW_BIN, NODE_BIN, commandExists, hostCommandEnv/, 'NemoClaw helper must use the shared host command resolver') assert.match(helperSource, /NEMOCLAW_BIN, NODE_BIN, commandExists, hostCommandEnv/, 'NemoClaw helper must use the shared host command resolver')
assert.match(helperSource, /parseVersion/, 'NemoClaw helper must parse CLI versions instead of assuming latest behavior') assert.match(helperSource, /parseVersion/, 'NemoClaw helper must parse CLI versions instead of assuming latest behavior')
assert.match(helperSource, /versionGte/, 'NemoClaw helper must compare semantic versions before using newer commands') assert.match(helperSource, /versionGte/, 'NemoClaw helper must compare semantic versions before using newer commands')
assert.match(helperSource, /export async function getNemoClawDoctorReport/, 'NemoClaw helper must expose a doctor JSON probe') assert.match(helperSource, /export async function getNemoClawDoctorReport/, 'NemoClaw helper must expose a doctor JSON probe')
assert.match(helperSource, /\[sandboxName, "doctor", "--json"\]/, 'doctor probe must use the structured NemoClaw doctor JSON command') assert.match(helperSource, /\[sandboxName, "doctor", "--json"\]/, 'doctor probe must use the structured NemoClaw doctor JSON command')
assert.match(helperSource, /export async function recoverSandboxWithNemoClaw/, 'NemoClaw helper must expose sandbox recovery') assert.match(helperSource, /export async function restartSandboxGatewayWithNemoClaw/, 'NemoClaw helper must expose native sandbox gateway restart')
assert.match(helperSource, /\[sandboxName, "recover"\]/, 'sandbox recovery must use the new NemoClaw recover command') assert.match(helperSource, /\["sandbox", "gateway", "restart", sandboxName\]/, 'sandbox restart must use the canonical NemoClaw native gateway restart command')
assert.match(helperSource, /supported: false,[\s\S]*attempted: false/, 'unsupported old NemoClaw versions must skip newer commands without failing') assert.match(helperSource, /supported: false,[\s\S]*attempted: false/, 'unsupported old NemoClaw versions must skip newer commands without failing')
assert.match(healthRouteSource, /getNemoClawDoctorReport/, 'sandbox health must include NemoClaw doctor integration when available') assert.match(healthRouteSource, /getNemoClawDoctorReport/, 'sandbox health must include NemoClaw doctor integration when available')
@@ -28,13 +28,9 @@ assert.match(healthRouteSource, /key: "nemoclaw-doctor"/, 'sandbox health must s
assert.match(healthRouteSource, /Using OpenShell compatibility checks/, 'sandbox health must explicitly fall back for current NemoClaw versions') assert.match(healthRouteSource, /Using OpenShell compatibility checks/, 'sandbox health must explicitly fall back for current NemoClaw versions')
assert.match(healthRouteSource, /minimumVersion/, 'sandbox health response must expose the minimum version for doctor support') assert.match(healthRouteSource, /minimumVersion/, 'sandbox health response must expose the minimum version for doctor support')
assert.match(restartRouteSource, /recoverSandboxWithNemoClaw/, 'sandbox restart must try NemoClaw recover when supported') assert.match(restartRouteSource, /restartSandboxGatewayWithNemoClaw/, 'sandbox restart must use NemoClaw native gateway restart when supported')
assert.ok( assert.match(restartRouteSource, /restartMode: "nemoclaw-native-gateway"/, 'restart route must report the native gateway lifecycle path')
restartRouteSource.indexOf('recoverSandboxWithNemoClaw(sandboxName)') < restartRouteSource.indexOf('runSandboxShell(sandboxName, restartOpenClawGatewayScript()'), assert.match(restartRouteSource, /did not bypass the agent-owned lifecycle boundary/, 'restart route must fail closed when native restart fails')
'sandbox restart must try NemoClaw recover before the legacy OpenClaw runtime restart fallback', assert.doesNotMatch(restartRouteSource, /runSandboxShell|restartOpenClawGatewayScript|openclaw gateway run/, 'sandbox restart must not bypass native agent lifecycle with a manual OpenClaw process launch')
)
assert.match(restartRouteSource, /restartMode: "nemoclaw-recover"/, 'restart route must report when the new NemoClaw recovery path succeeds')
assert.match(restartRouteSource, /restartMode: "openclaw-runtime"/, 'restart route must retain the current OpenShell/OpenClaw runtime fallback')
assert.match(restartRouteSource, /NemoClaw recover did not complete[\s\S]*fell back/, 'restart route must report fallback when modern recovery is attempted but fails')
console.log('nemoclaw-compat-check: PASS NemoClaw old/new compatibility assertions') console.log('nemoclaw-compat-check: PASS NemoClaw old/new compatibility assertions')
+2 -1
View File
@@ -307,7 +307,8 @@ assert.match(sandboxInferenceApplyLibSource, /\["inference", "set", "--no-verify
assert.match(sandboxInferenceApplyLibSource, /resolveInferenceModelIdentity/, 'live apply must normalize provider/model identities for OpenClaw inference routes') assert.match(sandboxInferenceApplyLibSource, /resolveInferenceModelIdentity/, 'live apply must normalize provider/model identities for OpenClaw inference routes')
assert.match(sandboxInferenceApplyLibSource, /providerKey: "nvidia"[\s\S]*modelRef: route\.model/, 'live apply must preserve nvidia/... model refs without wrapping them as inference/...') assert.match(sandboxInferenceApplyLibSource, /providerKey: "nvidia"[\s\S]*modelRef: route\.model/, 'live apply must preserve nvidia/... model refs without wrapping them as inference/...')
assert.doesNotMatch(sandboxRestartRouteSource, /delete", "pod"|kubectl delete/, 'restart route must not delete the backing sandbox pod') assert.doesNotMatch(sandboxRestartRouteSource, /delete", "pod"|kubectl delete/, 'restart route must not delete the backing sandbox pod')
assert.match(sandboxRestartRouteSource, /restartMode: "openclaw-runtime"/, 'restart route must restart only the in-sandbox OpenClaw runtime') assert.match(sandboxRestartRouteSource, /restartMode: "nemoclaw-native-gateway"/, 'restart route must use NemoClaw native agent gateway lifecycle')
assert.doesNotMatch(sandboxRestartRouteSource, /openclaw gateway run|runSandboxShell/, 'restart route must not bypass the native agent lifecycle with a manual OpenClaw launch')
assert.match(sandboxRestartRouteSource, /waitForSandboxReady\(sandboxName, 15000, 1000\)/, 'restart route must verify OpenShell reports the sandbox ready before runtime restart') assert.match(sandboxRestartRouteSource, /waitForSandboxReady\(sandboxName, 15000, 1000\)/, 'restart route must verify OpenShell reports the sandbox ready before runtime restart')
assert.match(sandboxListSource, /Restart Sandbox/, 'sandbox details must expose the Monday-era sandbox restart button') assert.match(sandboxListSource, /Restart Sandbox/, 'sandbox details must expose the Monday-era sandbox restart button')
assert.match(sandboxListSource, /\/api\/sandbox\/\$\{encodeURIComponent\(selectedSandbox\.id\)\}\/restart/, 'restart button must call the sandbox restart endpoint') assert.match(sandboxListSource, /\/api\/sandbox\/\$\{encodeURIComponent\(selectedSandbox\.id\)\}\/restart/, 'restart button must call the sandbox restart endpoint')