Handle gateway failures during sandbox delete

This commit is contained in:
Codex
2026-04-28 21:51:31 +00:00
parent 372326b225
commit bbade2e3e7
3 changed files with 31 additions and 7 deletions
+24 -3
View File
@@ -78,11 +78,28 @@ function isSandboxNotFound(output: string) {
return /sandbox not found|status:\s*NotFound|not present in the live OpenShell gateway/i.test(output) return /sandbox not found|status:\s*NotFound|not present in the live OpenShell gateway/i.test(output)
} }
function isGatewayRecoverable(output: string) {
return /Unknown gateway|Deploy it first|transport error|tcp connect error|Connection refused|invalid peer certificate|BadSignature/i.test(output)
}
function isValidNemoClawSandboxName(value: string) {
return /^[a-z][a-z0-9-]*[a-z0-9]$|^[a-z]$/.test(value)
}
function isNemoClawSandboxNotRegistered(output: string, sandboxName: string) { function isNemoClawSandboxNotRegistered(output: string, sandboxName: string) {
return new RegExp(`Unknown command:\\s*${sandboxName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`, "i").test(output) return new RegExp(`Unknown command:\\s*${sandboxName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`, "i").test(output)
} }
async function cleanupNemoClawSandbox(sandboxName: string) { async function cleanupNemoClawSandbox(sandboxName: string) {
if (!isValidNemoClawSandboxName(sandboxName)) {
return {
ok: true as const,
stdout: "",
stderr: "",
note: "Skipped NemoClaw registry cleanup because the target is not a valid NemoClaw sandbox name.",
}
}
const startedAt = Date.now() const startedAt = Date.now()
console.log(`[sandbox/delete] nemoclaw-cleanup:start sandbox=${sandboxName}`) console.log(`[sandbox/delete] nemoclaw-cleanup:start sandbox=${sandboxName}`)
try { try {
@@ -169,7 +186,6 @@ export async function POST(request: Request) {
const result = await deleteSandbox(target.sandboxName) const result = await deleteSandbox(target.sandboxName)
const deleteOutput = [result.error, result.stdout, result.stderr].filter(Boolean).join("\n") const deleteOutput = [result.error, result.stdout, result.stderr].filter(Boolean).join("\n")
const openShellAlreadyGone = !result.ok && isSandboxNotFound(deleteOutput) const openShellAlreadyGone = !result.ok && isSandboxNotFound(deleteOutput)
const cleanup = await cleanupNemoClawSandbox(target.sandboxName)
if (!result.ok && !openShellAlreadyGone) { if (!result.ok && !openShellAlreadyGone) {
return NextResponse.json({ return NextResponse.json({
@@ -183,10 +199,15 @@ export async function POST(request: Request) {
error: result.error, error: result.error,
stdout: result.stdout, stdout: result.stdout,
stderr: result.stderr, stderr: result.stderr,
nemoclaw: cleanup, recoverableGateway: isGatewayRecoverable(deleteOutput),
}, { status: 500 }) note: isGatewayRecoverable(deleteOutput)
? "OpenShell gateway is unavailable or its trust state is broken. Repair gateway trust, then retry delete."
: "OpenShell delete failed before NemoClaw cleanup ran.",
}, { status: isGatewayRecoverable(deleteOutput) ? 503 : 500 })
} }
const cleanup = await cleanupNemoClawSandbox(target.sandboxName)
if (!cleanup.ok) { if (!cleanup.ok) {
return NextResponse.json({ return NextResponse.json({
ok: false, ok: false,
+3 -3
View File
@@ -95,7 +95,7 @@ export default function Dashboard() {
const sleep = (ms: number) => new Promise((resolve) => window.setTimeout(resolve, ms)) const sleep = (ms: number) => new Promise((resolve) => window.setTimeout(resolve, ms))
const gatewayTrustError = Boolean(error && /BadSignature|invalid peer certificate|transport error|certificate signature/i.test(error)) const gatewayTrustError = Boolean(error && /BadSignature|invalid peer certificate|transport error|certificate signature|Unknown gateway|Deploy it first|Connection refused|tcp connect error/i.test(error))
const repairGatewayTrust = async () => { const repairGatewayTrust = async () => {
if (gatewayRepairing) return if (gatewayRepairing) return
@@ -346,7 +346,7 @@ export default function Dashboard() {
<div className="mx-auto mt-5 max-w-2xl rounded-sm border border-amber-500/40 bg-amber-500/10 p-4 text-left"> <div className="mx-auto mt-5 max-w-2xl rounded-sm border border-amber-500/40 bg-amber-500/10 p-4 text-left">
<h4 className="text-xs font-semibold uppercase tracking-wider text-amber-300">Gateway Trust Mismatch</h4> <h4 className="text-xs font-semibold uppercase tracking-wider text-amber-300">Gateway Trust Mismatch</h4>
<p className="mt-2 text-xs leading-5 text-amber-100/80"> <p className="mt-2 text-xs leading-5 text-amber-100/80">
OpenShell reached the gateway, but certificate verification failed. This usually means the gateway regenerated mTLS material while this controller still has stale trust files. OpenShell could not use the selected gateway. This usually means the gateway stopped, was undeployed, or regenerated mTLS material while this controller still has stale trust files.
</p> </p>
<button <button
type="button" type="button"
@@ -440,7 +440,7 @@ export default function Dashboard() {
<div className="mx-auto mt-5 max-w-2xl rounded-sm border border-amber-500/40 bg-amber-500/10 p-4 text-left"> <div className="mx-auto mt-5 max-w-2xl rounded-sm border border-amber-500/40 bg-amber-500/10 p-4 text-left">
<h4 className="text-xs font-semibold uppercase tracking-wider text-amber-300">Gateway Trust Mismatch</h4> <h4 className="text-xs font-semibold uppercase tracking-wider text-amber-300">Gateway Trust Mismatch</h4>
<p className="mt-2 text-xs leading-5 text-amber-100/80"> <p className="mt-2 text-xs leading-5 text-amber-100/80">
OpenShell reached the gateway, but certificate verification failed. This usually means the gateway regenerated mTLS material while this controller still has stale trust files. OpenShell could not use the selected gateway. This usually means the gateway stopped, was undeployed, or regenerated mTLS material while this controller still has stale trust files.
</p> </p>
<button <button
type="button" type="button"
+4 -1
View File
@@ -71,6 +71,9 @@ assert.match(deleteRouteSource, /cleanupNemoClawSandbox/, 'delete route must cle
assert.match(deleteRouteSource, /\[sandboxName, "destroy", "--yes"\]/, 'delete route must run the non-interactive NemoClaw destroy workflow') assert.match(deleteRouteSource, /\[sandboxName, "destroy", "--yes"\]/, 'delete route must run the non-interactive NemoClaw destroy workflow')
assert.match(deleteRouteSource, /openShellAlreadyGone/, 'delete route must tolerate sandboxes that OpenShell already deleted while NemoClaw still has registry state') assert.match(deleteRouteSource, /openShellAlreadyGone/, 'delete route must tolerate sandboxes that OpenShell already deleted while NemoClaw still has registry state')
assert.match(deleteRouteSource, /isNemoClawSandboxNotRegistered/, 'delete route must tolerate NemoClaw registry entries that were already cleaned up') assert.match(deleteRouteSource, /isNemoClawSandboxNotRegistered/, 'delete route must tolerate NemoClaw registry entries that were already cleaned up')
assert.match(deleteRouteSource, /isGatewayRecoverable/, 'delete route must detect recoverable OpenShell gateway failures')
assert.match(deleteRouteSource, /recoverableGateway/, 'delete failures must tell the UI when gateway repair can recover the operation')
assert.match(deleteRouteSource, /Skipped NemoClaw registry cleanup/, 'delete route must not pass UUID-style refs into NemoClaw cleanup')
assert.match(telemetryRouteSource, /const names = parseOpenShellSandboxNames\(sandboxListStdout\)/, 'telemetry must derive inventory from OpenShell before consulting NemoClaw') assert.match(telemetryRouteSource, /const names = parseOpenShellSandboxNames\(sandboxListStdout\)/, 'telemetry must derive inventory from OpenShell before consulting NemoClaw')
assert.match(telemetryRouteSource, /names\.length > 0[\s\S]*execNemoclaw\(\["list"\]\)[\s\S]*execNemoclaw\(\["status"\]\)/, 'telemetry must skip NemoClaw list/status when OpenShell reports zero live sandboxes') assert.match(telemetryRouteSource, /names\.length > 0[\s\S]*execNemoclaw\(\["list"\]\)[\s\S]*execNemoclaw\(\["status"\]\)/, 'telemetry must skip NemoClaw list/status when OpenShell reports zero live sandboxes')
assert.match(inferenceRouteSource, /\["provider", "create", "--name", name, "--type", type\]/, 'inference route must create OpenShell providers') assert.match(inferenceRouteSource, /\["provider", "create", "--name", name, "--type", type\]/, 'inference route must create OpenShell providers')
@@ -115,7 +118,7 @@ assert.match(sandboxListSource, /\/api\/sandbox\/\$\{encodeURIComponent\(selecte
assert.match(gatewayRepairRouteSource, /gateway", "start", "--name", gateway/, 'gateway repair route must restart the selected OpenShell gateway') assert.match(gatewayRepairRouteSource, /gateway", "start", "--name", gateway/, 'gateway repair route must restart the selected OpenShell gateway')
assert.match(gatewayRepairRouteSource, /backupOpenShellConfig/, 'gateway repair route must back up OpenShell config before repair') assert.match(gatewayRepairRouteSource, /backupOpenShellConfig/, 'gateway repair route must back up OpenShell config before repair')
assert.match(gatewayRepairRouteSource, /\["sandbox", "list"\]/, 'gateway repair route must verify sandbox inventory after repair') assert.match(gatewayRepairRouteSource, /\["sandbox", "list"\]/, 'gateway repair route must verify sandbox inventory after repair')
assert.match(pageSource, /BadSignature\|invalid peer certificate\|transport error\|certificate signature/, 'dashboard must detect gateway certificate trust failures') assert.match(pageSource, /BadSignature\|invalid peer certificate\|transport error\|certificate signature\|Unknown gateway\|Deploy it first\|Connection refused\|tcp connect error/, 'dashboard must detect gateway certificate and availability failures')
assert.match(pageSource, /Repair Gateway Trust/, 'inventory error UI must expose a guarded gateway trust repair button') assert.match(pageSource, /Repair Gateway Trust/, 'inventory error UI must expose a guarded gateway trust repair button')
assert.match(pageSource, /api\/openshell\/gateway\/repair/, 'gateway trust repair button must call the repair endpoint') assert.match(pageSource, /api\/openshell\/gateway\/repair/, 'gateway trust repair button must call the repair endpoint')
assert.match(pageSource, /It does not destroy sandboxes/, 'gateway trust repair warning must clarify sandboxes are not destroyed') assert.match(pageSource, /It does not destroy sandboxes/, 'gateway trust repair warning must clarify sandboxes are not destroyed')