fix: follow native agent gateway lifecycle (#49)
Co-authored-by: mmckeen-nv <mmckeen-nv@users.noreply.github.com>
This commit is contained in:
@@ -225,6 +225,8 @@ The dashboard shells out to the OpenShell CLI for several operations:
|
||||
|
||||
The **Quick Deploy New NemoClaw Sandbox** path clones a registered image through OpenShell, but it deliberately does not write the clone into NemoClaw's private `sandboxes.json`. Current NemoClaw entries include lifecycle identity, workload provenance, and agent-specific state that must be issued by NemoClaw itself. The clone remains visible and operable through OpenShell Control; use **Fresh NemoClaw Image** when the new sandbox must participate in NemoClaw-managed lifecycle operations.
|
||||
|
||||
Sandbox gateway restarts use NemoClaw's native-agent lifecycle command. The controller does not kill or launch OpenClaw or Hermes gateway processes directly; if NemoClaw cannot verify the restart, the request fails without bypassing the agent-owned lifecycle boundary.
|
||||
|
||||
OpenClaw dashboard access is loopback-only inside the host/sandbox context, so the UI uses local proxy routes:
|
||||
|
||||
- `/api/openshell/dashboard/proxy`
|
||||
|
||||
@@ -1,13 +1,7 @@
|
||||
import { NextResponse } from "next/server"
|
||||
import { execFile } from "node:child_process"
|
||||
import { promisify } from "node:util"
|
||||
import { OPENSHELL_BIN, hostCommandEnv } from "@/app/lib/hostCommands"
|
||||
import { recoverSandboxWithNemoClaw } from "@/app/lib/nemoclawCli"
|
||||
import { restartSandboxGatewayWithNemoClaw } from "@/app/lib/nemoclawCli"
|
||||
import { inspectSandbox, resolveSandboxRef } from "@/app/lib/openshellHost"
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const SANDBOX_DASHBOARD_REMOTE_PORT = Number.parseInt(process.env.OPENCLAW_SANDBOX_DASHBOARD_REMOTE_PORT || "18789", 10)
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
@@ -18,20 +12,6 @@ function validateSandboxName(value: string) {
|
||||
return value
|
||||
}
|
||||
|
||||
async function runOpenShell(args: string[], timeout = 30000) {
|
||||
const { stdout, stderr } = await execFileAsync(OPENSHELL_BIN, args, {
|
||||
env: hostCommandEnv({
|
||||
OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY?.trim() || undefined,
|
||||
}),
|
||||
timeout,
|
||||
maxBuffer: 4 * 1024 * 1024,
|
||||
})
|
||||
return { stdout: String(stdout).trim(), stderr: String(stderr).trim() }
|
||||
}
|
||||
|
||||
async function runSandboxShell(sandboxName: string, script: string, timeout = 30000) {
|
||||
return runOpenShell(["sandbox", "exec", "-n", sandboxName, "--", "sh", "-lc", script], timeout)
|
||||
}
|
||||
|
||||
async function waitForSandboxReady(sandboxName: string, timeoutMs: number, intervalMs: number) {
|
||||
const startedAt = Date.now()
|
||||
@@ -55,29 +35,6 @@ async function waitForSandboxReady(sandboxName: string, timeoutMs: number, inter
|
||||
return { ready: false, attempts, elapsedMs: Date.now() - startedAt, lastError }
|
||||
}
|
||||
|
||||
function restartOpenClawGatewayScript() {
|
||||
return [
|
||||
`port=${SANDBOX_DASHBOARD_REMOTE_PORT}`,
|
||||
"for p in /proc/[0-9]*; do",
|
||||
" cmd=$(tr '\\0' ' ' < \"$p/cmdline\" 2>/dev/null || true)",
|
||||
" case \"$cmd\" in",
|
||||
" *'openclaw gateway run'*) kill \"${p##*/}\" 2>/dev/null || true ;;",
|
||||
" esac",
|
||||
"done",
|
||||
"sleep 1",
|
||||
"if command -v openclaw >/dev/null 2>&1; then openclaw_bin=$(command -v openclaw);",
|
||||
"elif [ -x /usr/local/bin/openclaw ]; then openclaw_bin=/usr/local/bin/openclaw;",
|
||||
"else echo 'openclaw command not found in sandbox' >&2; exit 127; fi",
|
||||
"nohup \"$openclaw_bin\" gateway run --allow-unconfigured --bind loopback --port \"$port\" >/tmp/gateway.log 2>&1 &",
|
||||
"for i in 1 2 3 4 5 6 7 8 9 10; do",
|
||||
" curl -fsS --max-time 2 \"http://127.0.0.1:$port/\" >/dev/null 2>&1 && exit 0",
|
||||
" sleep 1",
|
||||
"done",
|
||||
"echo 'OpenClaw gateway did not answer after restart. Last log lines:' >&2",
|
||||
"tail -40 /tmp/gateway.log >&2 2>/dev/null || true",
|
||||
"exit 1",
|
||||
].join("\n")
|
||||
}
|
||||
|
||||
export async function POST(
|
||||
_request: Request,
|
||||
@@ -101,37 +58,34 @@ export async function POST(
|
||||
}, { status: 409 })
|
||||
}
|
||||
|
||||
const nemoclawRecover = await recoverSandboxWithNemoClaw(sandboxName)
|
||||
if (nemoclawRecover.attempted && nemoclawRecover.ok) {
|
||||
const nemoclawRestart = await restartSandboxGatewayWithNemoClaw(sandboxName)
|
||||
if (nemoclawRestart.attempted && nemoclawRestart.ok) {
|
||||
return NextResponse.json({
|
||||
ok: true,
|
||||
restarted: true,
|
||||
restartMode: "nemoclaw-recover",
|
||||
restartMode: "nemoclaw-native-gateway",
|
||||
sandboxId: resolved.id,
|
||||
sandboxName,
|
||||
readiness,
|
||||
nemoclawRecover,
|
||||
nemoclawRestart,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
note: "NemoClaw recover completed. The sandbox runtime and dashboard forward were checked without deleting the sandbox.",
|
||||
note: "NemoClaw asked the sandbox's native agent to restart its gateway, then verified gateway health and host forwards.",
|
||||
})
|
||||
}
|
||||
|
||||
const runtime = await runSandboxShell(sandboxName, restartOpenClawGatewayScript(), 45000)
|
||||
|
||||
return NextResponse.json({
|
||||
ok: true,
|
||||
restarted: true,
|
||||
restartMode: "openclaw-runtime",
|
||||
ok: false,
|
||||
restarted: false,
|
||||
restartMode: "nemoclaw-native-gateway",
|
||||
sandboxId: resolved.id,
|
||||
sandboxName,
|
||||
readiness,
|
||||
nemoclawRecover,
|
||||
runtime,
|
||||
nemoclawRestart,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
note: nemoclawRecover.attempted
|
||||
? "NemoClaw recover did not complete, so the dashboard fell back to restarting the in-sandbox OpenClaw runtime. The sandbox pod was not deleted."
|
||||
: "OpenClaw runtime restarted inside the sandbox. The sandbox pod was not deleted.",
|
||||
})
|
||||
note: nemoclawRestart.attempted
|
||||
? "NemoClaw could not complete the native gateway restart. The controller did not bypass the agent-owned lifecycle boundary."
|
||||
: "The installed NemoClaw CLI does not support the native gateway restart command. Upgrade NemoClaw before retrying.",
|
||||
}, { status: nemoclawRestart.attempted ? 502 : 409 })
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Failed to restart sandbox runtime"
|
||||
return NextResponse.json({ ok: false, restarted: false, error: message }, { status: /required|invalid/.test(message) ? 400 : 500 })
|
||||
|
||||
@@ -3,7 +3,7 @@ import { promisify } from "node:util"
|
||||
import { NEMOCLAW_BIN, NODE_BIN, commandExists, hostCommandEnv } from "./hostCommands"
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const NEMOCLAW_RECOVERY_MIN_VERSION = "0.0.37"
|
||||
const NEMOCLAW_LIFECYCLE_MIN_VERSION = "0.0.37"
|
||||
|
||||
type NemoClawInvocation = {
|
||||
file: string
|
||||
@@ -143,8 +143,8 @@ async function hasModernNemoClawSurface() {
|
||||
const version = await getNemoClawVersion()
|
||||
return {
|
||||
...version,
|
||||
supported: version.version ? versionGte(version.version, NEMOCLAW_RECOVERY_MIN_VERSION) : false,
|
||||
minimumVersion: NEMOCLAW_RECOVERY_MIN_VERSION,
|
||||
supported: version.version ? versionGte(version.version, NEMOCLAW_LIFECYCLE_MIN_VERSION) : false,
|
||||
minimumVersion: NEMOCLAW_LIFECYCLE_MIN_VERSION,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,7 +182,7 @@ export async function getNemoClawDoctorReport(sandboxName: string) {
|
||||
}
|
||||
}
|
||||
|
||||
export async function recoverSandboxWithNemoClaw(sandboxName: string) {
|
||||
export async function restartSandboxGatewayWithNemoClaw(sandboxName: string) {
|
||||
const capability = await hasModernNemoClawSurface()
|
||||
if (!capability.supported) {
|
||||
return {
|
||||
@@ -196,12 +196,12 @@ export async function recoverSandboxWithNemoClaw(sandboxName: string) {
|
||||
stderr: "",
|
||||
exitCode: null as number | null,
|
||||
error: capability.available
|
||||
? `NemoClaw ${capability.version || "unknown"} does not expose sandbox recover`
|
||||
? `NemoClaw ${capability.version || "unknown"} does not expose sandbox gateway restart`
|
||||
: capability.error || "NemoClaw CLI is not available",
|
||||
}
|
||||
}
|
||||
|
||||
const result = await runNemoClaw([sandboxName, "recover"], 90000)
|
||||
const result = await runNemoClaw(["sandbox", "gateway", "restart", sandboxName], 90000)
|
||||
return {
|
||||
available: true,
|
||||
supported: true,
|
||||
|
||||
@@ -13,14 +13,14 @@ const [helperSource, healthRouteSource, restartRouteSource] = await Promise.all(
|
||||
readFile(restartRoutePath, 'utf8'),
|
||||
])
|
||||
|
||||
assert.match(helperSource, /const NEMOCLAW_RECOVERY_MIN_VERSION = "0\.0\.37"/, 'NemoClaw doctor/recover integration must be gated behind the v0.0.37 CLI surface')
|
||||
assert.match(helperSource, /const NEMOCLAW_LIFECYCLE_MIN_VERSION = "0\.0\.37"/, 'NemoClaw doctor/lifecycle integration must be gated behind the v0.0.37 CLI surface')
|
||||
assert.match(helperSource, /NEMOCLAW_BIN, NODE_BIN, commandExists, hostCommandEnv/, 'NemoClaw helper must use the shared host command resolver')
|
||||
assert.match(helperSource, /parseVersion/, 'NemoClaw helper must parse CLI versions instead of assuming latest behavior')
|
||||
assert.match(helperSource, /versionGte/, 'NemoClaw helper must compare semantic versions before using newer commands')
|
||||
assert.match(helperSource, /export async function getNemoClawDoctorReport/, 'NemoClaw helper must expose a doctor JSON probe')
|
||||
assert.match(helperSource, /\[sandboxName, "doctor", "--json"\]/, 'doctor probe must use the structured NemoClaw doctor JSON command')
|
||||
assert.match(helperSource, /export async function recoverSandboxWithNemoClaw/, 'NemoClaw helper must expose sandbox recovery')
|
||||
assert.match(helperSource, /\[sandboxName, "recover"\]/, 'sandbox recovery must use the new NemoClaw recover command')
|
||||
assert.match(helperSource, /export async function restartSandboxGatewayWithNemoClaw/, 'NemoClaw helper must expose native sandbox gateway restart')
|
||||
assert.match(helperSource, /\["sandbox", "gateway", "restart", sandboxName\]/, 'sandbox restart must use the canonical NemoClaw native gateway restart command')
|
||||
assert.match(helperSource, /supported: false,[\s\S]*attempted: false/, 'unsupported old NemoClaw versions must skip newer commands without failing')
|
||||
|
||||
assert.match(healthRouteSource, /getNemoClawDoctorReport/, 'sandbox health must include NemoClaw doctor integration when available')
|
||||
@@ -28,13 +28,9 @@ assert.match(healthRouteSource, /key: "nemoclaw-doctor"/, 'sandbox health must s
|
||||
assert.match(healthRouteSource, /Using OpenShell compatibility checks/, 'sandbox health must explicitly fall back for current NemoClaw versions')
|
||||
assert.match(healthRouteSource, /minimumVersion/, 'sandbox health response must expose the minimum version for doctor support')
|
||||
|
||||
assert.match(restartRouteSource, /recoverSandboxWithNemoClaw/, 'sandbox restart must try NemoClaw recover when supported')
|
||||
assert.ok(
|
||||
restartRouteSource.indexOf('recoverSandboxWithNemoClaw(sandboxName)') < restartRouteSource.indexOf('runSandboxShell(sandboxName, restartOpenClawGatewayScript()'),
|
||||
'sandbox restart must try NemoClaw recover before the legacy OpenClaw runtime restart fallback',
|
||||
)
|
||||
assert.match(restartRouteSource, /restartMode: "nemoclaw-recover"/, 'restart route must report when the new NemoClaw recovery path succeeds')
|
||||
assert.match(restartRouteSource, /restartMode: "openclaw-runtime"/, 'restart route must retain the current OpenShell/OpenClaw runtime fallback')
|
||||
assert.match(restartRouteSource, /NemoClaw recover did not complete[\s\S]*fell back/, 'restart route must report fallback when modern recovery is attempted but fails')
|
||||
assert.match(restartRouteSource, /restartSandboxGatewayWithNemoClaw/, 'sandbox restart must use NemoClaw native gateway restart when supported')
|
||||
assert.match(restartRouteSource, /restartMode: "nemoclaw-native-gateway"/, 'restart route must report the native gateway lifecycle path')
|
||||
assert.match(restartRouteSource, /did not bypass the agent-owned lifecycle boundary/, 'restart route must fail closed when native restart fails')
|
||||
assert.doesNotMatch(restartRouteSource, /runSandboxShell|restartOpenClawGatewayScript|openclaw gateway run/, 'sandbox restart must not bypass native agent lifecycle with a manual OpenClaw process launch')
|
||||
|
||||
console.log('nemoclaw-compat-check: PASS NemoClaw old/new compatibility assertions')
|
||||
|
||||
@@ -307,7 +307,8 @@ assert.match(sandboxInferenceApplyLibSource, /\["inference", "set", "--no-verify
|
||||
assert.match(sandboxInferenceApplyLibSource, /resolveInferenceModelIdentity/, 'live apply must normalize provider/model identities for OpenClaw inference routes')
|
||||
assert.match(sandboxInferenceApplyLibSource, /providerKey: "nvidia"[\s\S]*modelRef: route\.model/, 'live apply must preserve nvidia/... model refs without wrapping them as inference/...')
|
||||
assert.doesNotMatch(sandboxRestartRouteSource, /delete", "pod"|kubectl delete/, 'restart route must not delete the backing sandbox pod')
|
||||
assert.match(sandboxRestartRouteSource, /restartMode: "openclaw-runtime"/, 'restart route must restart only the in-sandbox OpenClaw runtime')
|
||||
assert.match(sandboxRestartRouteSource, /restartMode: "nemoclaw-native-gateway"/, 'restart route must use NemoClaw native agent gateway lifecycle')
|
||||
assert.doesNotMatch(sandboxRestartRouteSource, /openclaw gateway run|runSandboxShell/, 'restart route must not bypass the native agent lifecycle with a manual OpenClaw launch')
|
||||
assert.match(sandboxRestartRouteSource, /waitForSandboxReady\(sandboxName, 15000, 1000\)/, 'restart route must verify OpenShell reports the sandbox ready before runtime restart')
|
||||
assert.match(sandboxListSource, /Restart Sandbox/, 'sandbox details must expose the Monday-era sandbox restart button')
|
||||
assert.match(sandboxListSource, /\/api\/sandbox\/\$\{encodeURIComponent\(selectedSandbox\.id\)\}\/restart/, 'restart button must call the sandbox restart endpoint')
|
||||
|
||||
Reference in New Issue
Block a user