Integrate NemoClaw recovery and dashboard routing

This commit is contained in:
mmckeen-nv
2026-05-08 09:32:56 -07:00
parent 819e3e5f7d
commit 94bd7737d3
25 changed files with 1001 additions and 152 deletions
+2 -2
View File
@@ -1,5 +1,5 @@
import { NextRequest, NextResponse } from "next/server"
import { createSessionCookieValue, getAuthSettings, sessionCookieOptions, verifyPassword } from "@/app/lib/controlAuth"
import { createSessionCookieValue, getAuthSettings, sessionCookieOptionsForRequest, verifyPassword } from "@/app/lib/controlAuth"
import { checkRateLimit, clearRateLimit, rateLimitKey, recordRateLimitFailure } from "@/app/lib/rateLimit"
const AUTH_WINDOW_MS = 5 * 60 * 1000
@@ -34,6 +34,6 @@ export async function POST(request: NextRequest) {
clearRateLimit(limitKey)
const response = NextResponse.json({ ok: true, next: nextPath })
response.cookies.set(settings.cookieName, await createSessionCookieValue(), sessionCookieOptions)
response.cookies.set(settings.cookieName, await createSessionCookieValue(), sessionCookieOptionsForRequest(request))
return response
}
+4 -7
View File
@@ -1,14 +1,11 @@
import { NextResponse } from "next/server"
import { getAuthSettings } from "@/app/lib/controlAuth"
import { NextRequest, NextResponse } from "next/server"
import { getAuthSettings, sessionCookieOptionsForRequest } from "@/app/lib/controlAuth"
export async function POST() {
export async function POST(request: NextRequest) {
const settings = getAuthSettings()
const response = NextResponse.json({ ok: true })
response.cookies.set(settings.cookieName, "", {
httpOnly: true,
sameSite: "lax",
secure: process.env.NODE_ENV === "production",
path: "/",
...sessionCookieOptionsForRequest(request),
maxAge: 0,
})
return response
+2 -2
View File
@@ -1,5 +1,5 @@
import { NextRequest, NextResponse } from "next/server"
import { createSessionCookieValue, getAuthSettings, sessionCookieOptions, verifyRecoveryToken } from "@/app/lib/controlAuth"
import { createSessionCookieValue, getAuthSettings, sessionCookieOptionsForRequest, verifyRecoveryToken } from "@/app/lib/controlAuth"
import { updateLocalAuthCredentials } from "@/app/lib/controlAuthConfig"
import { checkRateLimit, clearRateLimit, rateLimitKey, recordRateLimitFailure } from "@/app/lib/rateLimit"
@@ -37,6 +37,6 @@ export async function POST(request: NextRequest) {
recoveryToken: result.recoveryToken,
note: "Password reset. Save the new recovery token from .env.local.",
})
response.cookies.set(settings.cookieName, await createSessionCookieValue(), sessionCookieOptions)
response.cookies.set(settings.cookieName, await createSessionCookieValue(), sessionCookieOptionsForRequest(request))
return response
}
+2 -2
View File
@@ -2,7 +2,7 @@ import { NextRequest, NextResponse } from "next/server"
import {
createSessionCookieValue,
getAuthSettings,
sessionCookieOptions,
sessionCookieOptionsForRequest,
verifyPassword,
verifyRecoveryToken,
verifySessionCookieValue,
@@ -54,6 +54,6 @@ export async function POST(request: NextRequest) {
recoveryToken: result.recoveryToken,
note: "Password updated. Save the new recovery token from .env.local.",
})
response.cookies.set(settings.cookieName, await createSessionCookieValue(), sessionCookieOptions)
response.cookies.set(settings.cookieName, await createSessionCookieValue(), sessionCookieOptionsForRequest(request))
return response
}
+6 -2
View File
@@ -106,8 +106,12 @@ function isVllmEndpoint(name: string, baseUrl: string) {
return [name, baseUrl].some((value) => /vllm/i.test(value))
}
function isOllamaEndpoint(name: string, baseUrl: string) {
return [name, baseUrl].some((value) => /ollama|11434/i.test(value))
}
function normalizeProviderType(type: string, name: string, baseUrl: string) {
return isVllmEndpoint(name, baseUrl) ? "openai" : type
return isVllmEndpoint(name, baseUrl) || isOllamaEndpoint(name, baseUrl) ? "openai" : type
}
function redactSecretOutput(value: string, secrets: string[] = []) {
@@ -153,7 +157,7 @@ export async function POST(request: Request) {
const credentialKey = optionalString(body?.credentialKey) || "OPENAI_API_KEY"
const makeActive = body?.makeActive !== false
const system = body?.route === "system" || body?.system === true
const noVerify = body?.noVerify !== false || isVllmEndpoint(name, baseUrl)
const noVerify = body?.noVerify !== false || isVllmEndpoint(name, baseUrl) || isOllamaEndpoint(name, baseUrl)
const timeout = Number(body?.timeout ?? 0)
if (makeActive && !model) throw new Error("model is required when making the endpoint active")
+30 -2
View File
@@ -2,6 +2,7 @@ import { NextResponse } from 'next/server'
import { getDefaultOpenClawDashboardInstanceId, getOpenClawDashboardUrl, probeOpenClawDashboard } from '../../../../lib/openshellHost'
import { getOpenClawInstanceIdForSandbox, resolveOpenClawInstance, resolveOpenClawInstanceForSandbox } from '../../../../lib/openclawInstances'
import { resolveRuntimeAuthority } from '../../../../lib/runtimeAuthority'
import { extractOpenClawDashboardToken, setOpenClawDashboardTokenCookie } from '../../../../lib/openclawDashboardToken'
function parseInventoryCount(requestUrl: URL) {
const raw = requestUrl.searchParams.get('inventoryCount')
@@ -14,12 +15,37 @@ function firstHeaderValue(value: string | null) {
return value?.split(',')[0]?.trim() || null
}
function getConfiguredBaseWsUrl() {
return process.env.OPENCLAW_DASHBOARD_BASE_WS_URL?.trim() || process.env.BASE_WS_URL?.trim() || null
}
function normalizeBrowserWebSocketBase(value: string | null, proxiedUrl: string) {
if (!value) return null
const parsed = new URL(value)
if (parsed.protocol === 'http:') parsed.protocol = 'ws:'
if (parsed.protocol === 'https:') parsed.protocol = 'wss:'
if (parsed.protocol !== 'ws:' && parsed.protocol !== 'wss:') {
throw new Error('OPENCLAW_DASHBOARD_BASE_WS_URL/BASE_WS_URL must use ws://, wss://, http://, or https://')
}
parsed.hash = ''
if (parsed.pathname === '/' || parsed.pathname === '') {
parsed.pathname = proxiedUrl
} else {
parsed.pathname = parsed.pathname.replace(/\/+$/, '') || '/'
}
return parsed.toString()
}
function buildBrowserGatewayUrl(request: Request, requestUrl: URL, proxiedUrl: string) {
const configuredBaseWsUrl = normalizeBrowserWebSocketBase(getConfiguredBaseWsUrl(), proxiedUrl)
if (configuredBaseWsUrl) return configuredBaseWsUrl
const forwardedProtocol = firstHeaderValue(request.headers.get('x-forwarded-proto'))
const forwardedHost = firstHeaderValue(request.headers.get('x-forwarded-host'))
const host = forwardedHost || request.headers.get('host') || requestUrl.host
const protocol = (forwardedProtocol || requestUrl.protocol.replace(/:$/, '')) === 'https' ? 'wss:' : 'ws:'
const wsProxyPort = process.env.OPENCLAW_DASHBOARD_WS_PROXY_PORT?.trim() || '3001'
const wsProxyPort = process.env.OPENCLAW_DASHBOARD_WS_PROXY_PORT?.trim() || ''
const gatewayHost = wsProxyPort
? new URL(`${requestUrl.protocol}//${host}`).hostname + `:${wsProxyPort}`
: host
@@ -82,7 +108,7 @@ export async function GET(request: Request) {
const proxiedUrl = authority.bridgeActive ? instanceQualifiedProxyBase : '/api/openshell/dashboard/proxy'
const launchUrl = buildLaunchUrl(request, requestUrl, proxiedUrl, probe.bootstrapUrl)
return NextResponse.json({
const response = NextResponse.json({
ok: true,
sandboxId,
sandboxInstanceId: mappedSandboxInstanceId,
@@ -130,4 +156,6 @@ export async function GET(request: Request) {
? 'OpenClaw Dashboard is loopback-only on the host, so the web UI uses a local proxy route to expose it.'
: 'OpenClaw Dashboard is reachable directly for this instance.'
})
setOpenClawDashboardTokenCookie(response, request, proxiedUrl, extractOpenClawDashboardToken(probe.bootstrapUrl))
return response
}
+57 -6
View File
@@ -1,11 +1,13 @@
import { NextResponse } from 'next/server'
import { getDefaultOpenClawDashboardInstanceId } from '@/app/lib/openshellHost'
import { OPENCLAW_DASHBOARD_TOKEN_COOKIE } from '@/app/lib/openclawDashboardToken'
import { resolveRuntimeAuthority } from '@/app/lib/runtimeAuthority'
const LEGACY_PROXY_PREFIX = '/api/openshell/dashboard/proxy'
const INSTANCES_PREFIX = '/api/openshell/instances/'
const DASHBOARD_PROXY_SUFFIX = '/dashboard/proxy'
const BOOTSTRAP_SCRIPT_NAME = '__nemoclaw_openclaw_bootstrap.js'
const CONTROL_AUTH_COOKIE_NAME = 'openshell_control_session'
const HOP_BY_HOP_HEADERS = new Set([
'accept-encoding',
'connection',
@@ -95,6 +97,7 @@ function buildTargetUrl(requestUrl: URL) {
function copyRequestHeaders(request: Request, target: URL, controlUiOrigin: string) {
const headers = new Headers()
const dashboardToken = readCookieValue(request.headers.get('cookie'), OPENCLAW_DASHBOARD_TOKEN_COOKIE)
request.headers.forEach((value, key) => {
const lowerKey = key.toLowerCase()
@@ -104,16 +107,42 @@ function copyRequestHeaders(request: Request, target: URL, controlUiOrigin: stri
lowerKey !== 'origin' &&
lowerKey !== 'referer'
) {
headers.set(key, value)
const filteredValue = lowerKey === 'cookie' ? filterCookieHeader(value) : value
if (filteredValue) headers.set(key, filteredValue)
}
})
headers.set('host', target.host)
headers.set('origin', controlUiOrigin)
headers.set('referer', `${controlUiOrigin}/`)
if (dashboardToken && !headers.has('authorization')) {
headers.set('authorization', `Bearer ${dashboardToken}`)
}
return headers
}
function readCookieValue(value: string | null, name: string) {
return String(value || '')
.split(';')
.map((part) => part.trim())
.reduce<string | null>((found, part) => {
if (found || !part.startsWith(`${name}=`)) return found
try {
return decodeURIComponent(part.slice(name.length + 1))
} catch {
return part.slice(name.length + 1)
}
}, null)
}
function filterCookieHeader(value: string) {
return value
.split(';')
.map((part) => part.trim())
.filter((part) => part && !part.startsWith(`${CONTROL_AUTH_COOKIE_NAME}=`) && !part.startsWith(`${OPENCLAW_DASHBOARD_TOKEN_COOKIE}=`))
.join('; ')
}
function copyResponseHeaders(upstream: Response) {
const headers = new Headers()
@@ -151,24 +180,46 @@ function isBootstrapScriptRequest(requestUrl: URL, proxyPrefix: string) {
}
function bootstrapScriptResponse(proxyPrefix: string) {
const wsProxyPort = process.env.OPENCLAW_DASHBOARD_WS_PROXY_PORT?.trim() || '3001'
const baseWsUrl = process.env.OPENCLAW_DASHBOARD_BASE_WS_URL?.trim() || process.env.BASE_WS_URL?.trim() || ''
const wsProxyPort = process.env.OPENCLAW_DASHBOARD_WS_PROXY_PORT?.trim() || ''
const script = `
(() => {
const proxyPrefix = ${JSON.stringify(proxyPrefix)};
const baseWsUrl = ${JSON.stringify(baseWsUrl)};
const wsProxyPort = ${JSON.stringify(wsProxyPort)};
window.__OPENCLAW_CONTROL_UI_BASE_PATH__ = proxyPrefix;
try {
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
const pageGatewayUrl = protocol + '//' + window.location.host + proxyPrefix;
const sidecarHost = wsProxyPort ? window.location.hostname + ':' + wsProxyPort : window.location.host;
const sidecarGatewayUrl = protocol + '//' + sidecarHost + proxyPrefix;
const portGatewayUrl = wsProxyPort ? protocol + '//' + window.location.hostname + ':' + wsProxyPort + proxyPrefix : '';
const normalizeGatewayUrl = (value) => {
const raw = (value || '').trim();
if (!raw) return '';
try {
const parsed = new URL(raw, window.location.href);
if (parsed.protocol === 'http:') parsed.protocol = 'ws:';
if (parsed.protocol === 'https:') parsed.protocol = 'wss:';
if (parsed.protocol !== 'ws:' && parsed.protocol !== 'wss:') return '';
parsed.hash = '';
if (parsed.pathname === '/' || parsed.pathname === '') {
parsed.pathname = proxyPrefix;
} else {
parsed.pathname = parsed.pathname.replace(/\\/+$/, '') || '/';
}
return parsed.toString();
} catch {
return '';
}
};
const configuredGatewayUrl = normalizeGatewayUrl(baseWsUrl);
const defaultGatewayUrl = configuredGatewayUrl || portGatewayUrl || pageGatewayUrl;
const settingsKey = 'openclaw.control.settings.v1';
const settingsPrefix = 'openclaw.control.settings.v1:';
const tokenKey = 'openclaw.control.token.v1';
const tokenPrefix = 'openclaw.control.token.v1:';
const hashParams = new URLSearchParams(window.location.hash.startsWith('#') ? window.location.hash.slice(1) : window.location.hash);
const gatewayUrl = (hashParams.get('gatewayUrl') || '').trim() || sidecarGatewayUrl;
const gatewayUrl = normalizeGatewayUrl(hashParams.get('gatewayUrl')) || defaultGatewayUrl;
const token = (hashParams.get('token') || '').trim();
const storageScope = (value) => {
try {
@@ -180,7 +231,7 @@ function bootstrapScriptResponse(proxyPrefix: string) {
}
};
const uniqueScopes = (values) => Array.from(new Set(values.map(storageScope).filter(Boolean)));
const gatewayScopes = uniqueScopes([gatewayUrl, sidecarGatewayUrl, pageGatewayUrl]);
const gatewayScopes = uniqueScopes([gatewayUrl, configuredGatewayUrl, portGatewayUrl, pageGatewayUrl]);
const settingsKeys = gatewayScopes.map((scope) => settingsPrefix + scope);
const readSettings = () => {
for (const key of [...settingsKeys, settingsPrefix + 'default', settingsKey]) {
@@ -1,6 +1,7 @@
import { NextResponse } from "next/server"
import { execOpenShell, normalizeSandboxPhase, resolveSandboxRef } from "@/app/lib/openshellHost"
import { listBackupCatalog } from "@/app/lib/backupCatalog"
import { getNemoClawDoctorReport } from "@/app/lib/nemoclawCli"
function parseField(output: string, label: string) {
const normalizedLabel = label.toLowerCase()
@@ -23,6 +24,15 @@ export async function GET(
const phase = normalizeSandboxPhase(parseField(resolved.details, "Phase"))
const catalog = await listBackupCatalog()
const relatedBackups = catalog.filter((backup) => backup.sandboxId === sandboxId || backup.sandboxName === resolved.name)
const nemoclawDoctor = await getNemoClawDoctorReport(resolved.name).catch((error) => ({
available: false,
supported: false,
attempted: false,
version: null,
minimumVersion: "0.0.37",
report: null,
error: error instanceof Error ? error.message : "NemoClaw doctor check failed",
}))
let sshConfig = ""
let sshConfigError = ""
@@ -60,6 +70,16 @@ export async function GET(
? `${relatedBackups.length} saved backup${relatedBackups.length === 1 ? "" : "s"} in catalog.`
: "No saved catalog backups yet.",
},
{
key: "nemoclaw-doctor",
label: "NemoClaw doctor",
ok: !nemoclawDoctor.supported || Boolean(nemoclawDoctor.report && nemoclawDoctor.report.status !== "fail"),
detail: nemoclawDoctor.supported
? nemoclawDoctor.report
? `NemoClaw ${nemoclawDoctor.version || "unknown"} doctor reported ${nemoclawDoctor.report.status} (${nemoclawDoctor.report.failed} failed, ${nemoclawDoctor.report.warnings} warnings).`
: nemoclawDoctor.error || "NemoClaw doctor did not return a report."
: `Using OpenShell compatibility checks; NemoClaw doctor requires ${nemoclawDoctor.minimumVersion}+${nemoclawDoctor.version ? `, found ${nemoclawDoctor.version}` : ""}.`,
},
]
return NextResponse.json({
@@ -71,6 +91,19 @@ export async function GET(
phase,
},
checks,
nemoclaw: {
doctor: {
available: nemoclawDoctor.available,
supported: nemoclawDoctor.supported,
attempted: nemoclawDoctor.attempted,
version: nemoclawDoctor.version,
minimumVersion: nemoclawDoctor.minimumVersion,
status: nemoclawDoctor.report?.status ?? null,
failed: nemoclawDoctor.report?.failed ?? null,
warnings: nemoclawDoctor.report?.warnings ?? null,
error: nemoclawDoctor.error ?? null,
},
},
backupCount: relatedBackups.length,
durationMs: Date.now() - startedAt,
checkedAt: new Date().toISOString(),
+20 -1
View File
@@ -2,6 +2,7 @@ import { NextResponse } from "next/server"
import { execFile } from "node:child_process"
import { promisify } from "node:util"
import { OPENSHELL_BIN, hostCommandEnv } from "@/app/lib/hostCommands"
import { recoverSandboxWithNemoClaw } from "@/app/lib/nemoclawCli"
import { inspectSandbox, resolveSandboxRef } from "@/app/lib/openshellHost"
const execFileAsync = promisify(execFile)
@@ -100,6 +101,21 @@ export async function POST(
}, { status: 409 })
}
const nemoclawRecover = await recoverSandboxWithNemoClaw(sandboxName)
if (nemoclawRecover.attempted && nemoclawRecover.ok) {
return NextResponse.json({
ok: true,
restarted: true,
restartMode: "nemoclaw-recover",
sandboxId: resolved.id,
sandboxName,
readiness,
nemoclawRecover,
elapsedMs: Date.now() - startedAt,
note: "NemoClaw recover completed. The sandbox runtime and dashboard forward were checked without deleting the sandbox.",
})
}
const runtime = await runSandboxShell(sandboxName, restartOpenClawGatewayScript(), 45000)
return NextResponse.json({
@@ -109,9 +125,12 @@ export async function POST(
sandboxId: resolved.id,
sandboxName,
readiness,
nemoclawRecover,
runtime,
elapsedMs: Date.now() - startedAt,
note: "OpenClaw runtime restarted inside the sandbox. The sandbox pod was not deleted.",
note: nemoclawRecover.attempted
? "NemoClaw recover did not complete, so the dashboard fell back to restarting the in-sandbox OpenClaw runtime. The sandbox pod was not deleted."
: "OpenClaw runtime restarted inside the sandbox. The sandbox pod was not deleted.",
})
} catch (error) {
const message = error instanceof Error ? error.message : "Failed to restart sandbox runtime"