Handle gateway failures during sandbox delete
This commit is contained in:
@@ -78,11 +78,28 @@ function isSandboxNotFound(output: string) {
|
|||||||
return /sandbox not found|status:\s*NotFound|not present in the live OpenShell gateway/i.test(output)
|
return /sandbox not found|status:\s*NotFound|not present in the live OpenShell gateway/i.test(output)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isGatewayRecoverable(output: string) {
|
||||||
|
return /Unknown gateway|Deploy it first|transport error|tcp connect error|Connection refused|invalid peer certificate|BadSignature/i.test(output)
|
||||||
|
}
|
||||||
|
|
||||||
|
function isValidNemoClawSandboxName(value: string) {
|
||||||
|
return /^[a-z][a-z0-9-]*[a-z0-9]$|^[a-z]$/.test(value)
|
||||||
|
}
|
||||||
|
|
||||||
function isNemoClawSandboxNotRegistered(output: string, sandboxName: string) {
|
function isNemoClawSandboxNotRegistered(output: string, sandboxName: string) {
|
||||||
return new RegExp(`Unknown command:\\s*${sandboxName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`, "i").test(output)
|
return new RegExp(`Unknown command:\\s*${sandboxName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`, "i").test(output)
|
||||||
}
|
}
|
||||||
|
|
||||||
async function cleanupNemoClawSandbox(sandboxName: string) {
|
async function cleanupNemoClawSandbox(sandboxName: string) {
|
||||||
|
if (!isValidNemoClawSandboxName(sandboxName)) {
|
||||||
|
return {
|
||||||
|
ok: true as const,
|
||||||
|
stdout: "",
|
||||||
|
stderr: "",
|
||||||
|
note: "Skipped NemoClaw registry cleanup because the target is not a valid NemoClaw sandbox name.",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const startedAt = Date.now()
|
const startedAt = Date.now()
|
||||||
console.log(`[sandbox/delete] nemoclaw-cleanup:start sandbox=${sandboxName}`)
|
console.log(`[sandbox/delete] nemoclaw-cleanup:start sandbox=${sandboxName}`)
|
||||||
try {
|
try {
|
||||||
@@ -169,7 +186,6 @@ export async function POST(request: Request) {
|
|||||||
const result = await deleteSandbox(target.sandboxName)
|
const result = await deleteSandbox(target.sandboxName)
|
||||||
const deleteOutput = [result.error, result.stdout, result.stderr].filter(Boolean).join("\n")
|
const deleteOutput = [result.error, result.stdout, result.stderr].filter(Boolean).join("\n")
|
||||||
const openShellAlreadyGone = !result.ok && isSandboxNotFound(deleteOutput)
|
const openShellAlreadyGone = !result.ok && isSandboxNotFound(deleteOutput)
|
||||||
const cleanup = await cleanupNemoClawSandbox(target.sandboxName)
|
|
||||||
|
|
||||||
if (!result.ok && !openShellAlreadyGone) {
|
if (!result.ok && !openShellAlreadyGone) {
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
@@ -183,10 +199,15 @@ export async function POST(request: Request) {
|
|||||||
error: result.error,
|
error: result.error,
|
||||||
stdout: result.stdout,
|
stdout: result.stdout,
|
||||||
stderr: result.stderr,
|
stderr: result.stderr,
|
||||||
nemoclaw: cleanup,
|
recoverableGateway: isGatewayRecoverable(deleteOutput),
|
||||||
}, { status: 500 })
|
note: isGatewayRecoverable(deleteOutput)
|
||||||
|
? "OpenShell gateway is unavailable or its trust state is broken. Repair gateway trust, then retry delete."
|
||||||
|
: "OpenShell delete failed before NemoClaw cleanup ran.",
|
||||||
|
}, { status: isGatewayRecoverable(deleteOutput) ? 503 : 500 })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const cleanup = await cleanupNemoClawSandbox(target.sandboxName)
|
||||||
|
|
||||||
if (!cleanup.ok) {
|
if (!cleanup.ok) {
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
ok: false,
|
ok: false,
|
||||||
|
|||||||
+3
-3
@@ -95,7 +95,7 @@ export default function Dashboard() {
|
|||||||
|
|
||||||
const sleep = (ms: number) => new Promise((resolve) => window.setTimeout(resolve, ms))
|
const sleep = (ms: number) => new Promise((resolve) => window.setTimeout(resolve, ms))
|
||||||
|
|
||||||
const gatewayTrustError = Boolean(error && /BadSignature|invalid peer certificate|transport error|certificate signature/i.test(error))
|
const gatewayTrustError = Boolean(error && /BadSignature|invalid peer certificate|transport error|certificate signature|Unknown gateway|Deploy it first|Connection refused|tcp connect error/i.test(error))
|
||||||
|
|
||||||
const repairGatewayTrust = async () => {
|
const repairGatewayTrust = async () => {
|
||||||
if (gatewayRepairing) return
|
if (gatewayRepairing) return
|
||||||
@@ -346,7 +346,7 @@ export default function Dashboard() {
|
|||||||
<div className="mx-auto mt-5 max-w-2xl rounded-sm border border-amber-500/40 bg-amber-500/10 p-4 text-left">
|
<div className="mx-auto mt-5 max-w-2xl rounded-sm border border-amber-500/40 bg-amber-500/10 p-4 text-left">
|
||||||
<h4 className="text-xs font-semibold uppercase tracking-wider text-amber-300">Gateway Trust Mismatch</h4>
|
<h4 className="text-xs font-semibold uppercase tracking-wider text-amber-300">Gateway Trust Mismatch</h4>
|
||||||
<p className="mt-2 text-xs leading-5 text-amber-100/80">
|
<p className="mt-2 text-xs leading-5 text-amber-100/80">
|
||||||
OpenShell reached the gateway, but certificate verification failed. This usually means the gateway regenerated mTLS material while this controller still has stale trust files.
|
OpenShell could not use the selected gateway. This usually means the gateway stopped, was undeployed, or regenerated mTLS material while this controller still has stale trust files.
|
||||||
</p>
|
</p>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
@@ -440,7 +440,7 @@ export default function Dashboard() {
|
|||||||
<div className="mx-auto mt-5 max-w-2xl rounded-sm border border-amber-500/40 bg-amber-500/10 p-4 text-left">
|
<div className="mx-auto mt-5 max-w-2xl rounded-sm border border-amber-500/40 bg-amber-500/10 p-4 text-left">
|
||||||
<h4 className="text-xs font-semibold uppercase tracking-wider text-amber-300">Gateway Trust Mismatch</h4>
|
<h4 className="text-xs font-semibold uppercase tracking-wider text-amber-300">Gateway Trust Mismatch</h4>
|
||||||
<p className="mt-2 text-xs leading-5 text-amber-100/80">
|
<p className="mt-2 text-xs leading-5 text-amber-100/80">
|
||||||
OpenShell reached the gateway, but certificate verification failed. This usually means the gateway regenerated mTLS material while this controller still has stale trust files.
|
OpenShell could not use the selected gateway. This usually means the gateway stopped, was undeployed, or regenerated mTLS material while this controller still has stale trust files.
|
||||||
</p>
|
</p>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
|
|||||||
@@ -71,6 +71,9 @@ assert.match(deleteRouteSource, /cleanupNemoClawSandbox/, 'delete route must cle
|
|||||||
assert.match(deleteRouteSource, /\[sandboxName, "destroy", "--yes"\]/, 'delete route must run the non-interactive NemoClaw destroy workflow')
|
assert.match(deleteRouteSource, /\[sandboxName, "destroy", "--yes"\]/, 'delete route must run the non-interactive NemoClaw destroy workflow')
|
||||||
assert.match(deleteRouteSource, /openShellAlreadyGone/, 'delete route must tolerate sandboxes that OpenShell already deleted while NemoClaw still has registry state')
|
assert.match(deleteRouteSource, /openShellAlreadyGone/, 'delete route must tolerate sandboxes that OpenShell already deleted while NemoClaw still has registry state')
|
||||||
assert.match(deleteRouteSource, /isNemoClawSandboxNotRegistered/, 'delete route must tolerate NemoClaw registry entries that were already cleaned up')
|
assert.match(deleteRouteSource, /isNemoClawSandboxNotRegistered/, 'delete route must tolerate NemoClaw registry entries that were already cleaned up')
|
||||||
|
assert.match(deleteRouteSource, /isGatewayRecoverable/, 'delete route must detect recoverable OpenShell gateway failures')
|
||||||
|
assert.match(deleteRouteSource, /recoverableGateway/, 'delete failures must tell the UI when gateway repair can recover the operation')
|
||||||
|
assert.match(deleteRouteSource, /Skipped NemoClaw registry cleanup/, 'delete route must not pass UUID-style refs into NemoClaw cleanup')
|
||||||
assert.match(telemetryRouteSource, /const names = parseOpenShellSandboxNames\(sandboxListStdout\)/, 'telemetry must derive inventory from OpenShell before consulting NemoClaw')
|
assert.match(telemetryRouteSource, /const names = parseOpenShellSandboxNames\(sandboxListStdout\)/, 'telemetry must derive inventory from OpenShell before consulting NemoClaw')
|
||||||
assert.match(telemetryRouteSource, /names\.length > 0[\s\S]*execNemoclaw\(\["list"\]\)[\s\S]*execNemoclaw\(\["status"\]\)/, 'telemetry must skip NemoClaw list/status when OpenShell reports zero live sandboxes')
|
assert.match(telemetryRouteSource, /names\.length > 0[\s\S]*execNemoclaw\(\["list"\]\)[\s\S]*execNemoclaw\(\["status"\]\)/, 'telemetry must skip NemoClaw list/status when OpenShell reports zero live sandboxes')
|
||||||
assert.match(inferenceRouteSource, /\["provider", "create", "--name", name, "--type", type\]/, 'inference route must create OpenShell providers')
|
assert.match(inferenceRouteSource, /\["provider", "create", "--name", name, "--type", type\]/, 'inference route must create OpenShell providers')
|
||||||
@@ -115,7 +118,7 @@ assert.match(sandboxListSource, /\/api\/sandbox\/\$\{encodeURIComponent\(selecte
|
|||||||
assert.match(gatewayRepairRouteSource, /gateway", "start", "--name", gateway/, 'gateway repair route must restart the selected OpenShell gateway')
|
assert.match(gatewayRepairRouteSource, /gateway", "start", "--name", gateway/, 'gateway repair route must restart the selected OpenShell gateway')
|
||||||
assert.match(gatewayRepairRouteSource, /backupOpenShellConfig/, 'gateway repair route must back up OpenShell config before repair')
|
assert.match(gatewayRepairRouteSource, /backupOpenShellConfig/, 'gateway repair route must back up OpenShell config before repair')
|
||||||
assert.match(gatewayRepairRouteSource, /\["sandbox", "list"\]/, 'gateway repair route must verify sandbox inventory after repair')
|
assert.match(gatewayRepairRouteSource, /\["sandbox", "list"\]/, 'gateway repair route must verify sandbox inventory after repair')
|
||||||
assert.match(pageSource, /BadSignature\|invalid peer certificate\|transport error\|certificate signature/, 'dashboard must detect gateway certificate trust failures')
|
assert.match(pageSource, /BadSignature\|invalid peer certificate\|transport error\|certificate signature\|Unknown gateway\|Deploy it first\|Connection refused\|tcp connect error/, 'dashboard must detect gateway certificate and availability failures')
|
||||||
assert.match(pageSource, /Repair Gateway Trust/, 'inventory error UI must expose a guarded gateway trust repair button')
|
assert.match(pageSource, /Repair Gateway Trust/, 'inventory error UI must expose a guarded gateway trust repair button')
|
||||||
assert.match(pageSource, /api\/openshell\/gateway\/repair/, 'gateway trust repair button must call the repair endpoint')
|
assert.match(pageSource, /api\/openshell\/gateway\/repair/, 'gateway trust repair button must call the repair endpoint')
|
||||||
assert.match(pageSource, /It does not destroy sandboxes/, 'gateway trust repair warning must clarify sandboxes are not destroyed')
|
assert.match(pageSource, /It does not destroy sandboxes/, 'gateway trust repair warning must clarify sandboxes are not destroyed')
|
||||||
|
|||||||
Reference in New Issue
Block a user