Use configured inference model for MCP assists
This commit is contained in:
@@ -1,8 +1,7 @@
|
|||||||
import { NextResponse } from "next/server"
|
import { NextResponse } from "next/server"
|
||||||
|
import { resolveOpenAiCompatibleBaseUrl, resolvePrimaryInferenceModel } from "@/app/lib/inferenceModel"
|
||||||
|
|
||||||
const REQUEST_TIMEOUT_MS = Number.parseInt(process.env.MCP_INSTALL_ASSIST_TIMEOUT_MS || "45000", 10)
|
const REQUEST_TIMEOUT_MS = Number.parseInt(process.env.MCP_INSTALL_ASSIST_TIMEOUT_MS || "45000", 10)
|
||||||
const MODEL = process.env.MCP_PREFLIGHT_LLM_MODEL || process.env.OPENAI_MODEL || "gpt-4o-mini"
|
|
||||||
const BASE_URL = (process.env.MCP_PREFLIGHT_LLM_BASE_URL || process.env.OPENAI_BASE_URL || process.env.VLLM_BASE_URL || "http://localhost:8000/v1").replace(/\/+$/, "")
|
|
||||||
|
|
||||||
function extractJsonObject(value: string) {
|
function extractJsonObject(value: string) {
|
||||||
const trimmed = value.trim()
|
const trimmed = value.trim()
|
||||||
@@ -34,8 +33,10 @@ export async function POST(request: Request) {
|
|||||||
const body = await request.json().catch(() => ({}))
|
const body = await request.json().catch(() => ({}))
|
||||||
const prompt = typeof body?.prompt === "string" ? body.prompt.trim() : ""
|
const prompt = typeof body?.prompt === "string" ? body.prompt.trim() : ""
|
||||||
if (!prompt) throw new Error("install prompt is required")
|
if (!prompt) throw new Error("install prompt is required")
|
||||||
|
const model = await resolvePrimaryInferenceModel()
|
||||||
|
if (!model) throw new Error("No primary inference model is configured")
|
||||||
|
|
||||||
const response = await fetch(`${BASE_URL}/chat/completions`, {
|
const response = await fetch(`${resolveOpenAiCompatibleBaseUrl()}/chat/completions`, {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
signal: controller.signal,
|
signal: controller.signal,
|
||||||
headers: {
|
headers: {
|
||||||
@@ -45,7 +46,7 @@ export async function POST(request: Request) {
|
|||||||
: {}),
|
: {}),
|
||||||
},
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
model: MODEL,
|
model,
|
||||||
temperature: 0.1,
|
temperature: 0.1,
|
||||||
response_format: { type: "json_object" },
|
response_format: { type: "json_object" },
|
||||||
messages: [
|
messages: [
|
||||||
@@ -88,7 +89,7 @@ export async function POST(request: Request) {
|
|||||||
uploadEntrypoint: typeof suggestion?.uploadEntrypoint === "string" ? suggestion.uploadEntrypoint : "",
|
uploadEntrypoint: typeof suggestion?.uploadEntrypoint === "string" ? suggestion.uploadEntrypoint : "",
|
||||||
notes: typeof suggestion?.notes === "string" ? suggestion.notes : "",
|
notes: typeof suggestion?.notes === "string" ? suggestion.notes : "",
|
||||||
},
|
},
|
||||||
model: MODEL,
|
model,
|
||||||
})
|
})
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const message = error instanceof Error ? error.message : "Failed to assist MCP install"
|
const message = error instanceof Error ? error.message : "Failed to assist MCP install"
|
||||||
|
|||||||
@@ -1,8 +1,7 @@
|
|||||||
import { NextResponse } from "next/server"
|
import { NextResponse } from "next/server"
|
||||||
|
import { resolveOpenAiCompatibleBaseUrl, resolvePrimaryInferenceModel } from "@/app/lib/inferenceModel"
|
||||||
|
|
||||||
const REQUEST_TIMEOUT_MS = Number.parseInt(process.env.MCP_REGISTRY_ASSIST_TIMEOUT_MS || "45000", 10)
|
const REQUEST_TIMEOUT_MS = Number.parseInt(process.env.MCP_REGISTRY_ASSIST_TIMEOUT_MS || "45000", 10)
|
||||||
const MODEL = process.env.MCP_PREFLIGHT_LLM_MODEL || process.env.OPENAI_MODEL || "gpt-4o-mini"
|
|
||||||
const BASE_URL = (process.env.MCP_PREFLIGHT_LLM_BASE_URL || process.env.OPENAI_BASE_URL || process.env.VLLM_BASE_URL || "http://localhost:8000/v1").replace(/\/+$/, "")
|
|
||||||
|
|
||||||
function extractJsonObject(value: string) {
|
function extractJsonObject(value: string) {
|
||||||
const trimmed = value.trim()
|
const trimmed = value.trim()
|
||||||
@@ -19,8 +18,10 @@ export async function POST(request: Request) {
|
|||||||
const body = await request.json().catch(() => ({}))
|
const body = await request.json().catch(() => ({}))
|
||||||
const prompt = typeof body?.prompt === "string" ? body.prompt.trim() : ""
|
const prompt = typeof body?.prompt === "string" ? body.prompt.trim() : ""
|
||||||
if (!prompt) throw new Error("registry prompt is required")
|
if (!prompt) throw new Error("registry prompt is required")
|
||||||
|
const model = await resolvePrimaryInferenceModel()
|
||||||
|
if (!model) throw new Error("No primary inference model is configured")
|
||||||
|
|
||||||
const response = await fetch(`${BASE_URL}/chat/completions`, {
|
const response = await fetch(`${resolveOpenAiCompatibleBaseUrl()}/chat/completions`, {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
signal: controller.signal,
|
signal: controller.signal,
|
||||||
headers: {
|
headers: {
|
||||||
@@ -30,7 +31,7 @@ export async function POST(request: Request) {
|
|||||||
: {}),
|
: {}),
|
||||||
},
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
model: MODEL,
|
model,
|
||||||
temperature: 0.1,
|
temperature: 0.1,
|
||||||
response_format: { type: "json_object" },
|
response_format: { type: "json_object" },
|
||||||
messages: [
|
messages: [
|
||||||
@@ -56,7 +57,7 @@ export async function POST(request: Request) {
|
|||||||
baseUrl: typeof suggestion?.baseUrl === "string" ? suggestion.baseUrl : "",
|
baseUrl: typeof suggestion?.baseUrl === "string" ? suggestion.baseUrl : "",
|
||||||
description: typeof suggestion?.description === "string" ? suggestion.description : "",
|
description: typeof suggestion?.description === "string" ? suggestion.description : "",
|
||||||
},
|
},
|
||||||
model: MODEL,
|
model,
|
||||||
})
|
})
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const message = error instanceof Error ? error.message : "Failed to assist registry setup"
|
const message = error instanceof Error ? error.message : "Failed to assist registry setup"
|
||||||
|
|||||||
@@ -373,7 +373,7 @@ export async function POST(request: Request) {
|
|||||||
attempted: true,
|
attempted: true,
|
||||||
ok: false,
|
ok: false,
|
||||||
provider: "openai-compatible",
|
provider: "openai-compatible",
|
||||||
model: process.env.MCP_PREFLIGHT_LLM_MODEL || process.env.OPENAI_MODEL || "",
|
model: "",
|
||||||
baseUrl: process.env.MCP_PREFLIGHT_LLM_BASE_URL || process.env.OPENAI_BASE_URL || process.env.VLLM_BASE_URL || "",
|
baseUrl: process.env.MCP_PREFLIGHT_LLM_BASE_URL || process.env.OPENAI_BASE_URL || process.env.VLLM_BASE_URL || "",
|
||||||
summary: "LLM-assisted repair could not complete.",
|
summary: "LLM-assisted repair could not complete.",
|
||||||
changes: [],
|
changes: [],
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
import { execFile } from "node:child_process"
|
||||||
|
import { promisify } from "node:util"
|
||||||
|
import { OPENSHELL_BIN, hostCommandEnv } from "./hostCommands"
|
||||||
|
|
||||||
|
const execFileAsync = promisify(execFile)
|
||||||
|
|
||||||
|
function stripAnsi(value: string) {
|
||||||
|
return value.replace(/\x1B\[[0-?]*[ -/]*[@-~]/g, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
function readField(output: string, label: string) {
|
||||||
|
const match = output.match(new RegExp(`^\\s*${label}:\\s*(.+?)\\s*$`, "im"))
|
||||||
|
return match ? match[1].trim() : ""
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseRouteModel(output: string, label: "Gateway inference" | "System inference") {
|
||||||
|
const otherLabel = label === "Gateway inference" ? "System inference" : "Gateway inference"
|
||||||
|
const pattern = new RegExp(`${label}:\\s*([\\s\\S]*?)(?:\\n\\s*${otherLabel}:|$)`, "i")
|
||||||
|
const section = output.match(pattern)?.[1] ?? ""
|
||||||
|
if (!section || /Not configured/i.test(section)) return ""
|
||||||
|
return readField(section, "Model")
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readOpenShellInferenceModel() {
|
||||||
|
const { stdout } = await execFileAsync(OPENSHELL_BIN, ["inference", "get"], {
|
||||||
|
env: hostCommandEnv({
|
||||||
|
OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY || "nemoclaw",
|
||||||
|
}),
|
||||||
|
timeout: 30000,
|
||||||
|
maxBuffer: 5 * 1024 * 1024,
|
||||||
|
})
|
||||||
|
const output = stripAnsi(String(stdout).trim())
|
||||||
|
return parseRouteModel(output, "Gateway inference") || parseRouteModel(output, "System inference")
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function resolvePrimaryInferenceModel(explicitModel?: string | null) {
|
||||||
|
if (explicitModel?.trim()) return explicitModel.trim()
|
||||||
|
if (process.env.MCP_PREFLIGHT_LLM_MODEL?.trim()) return process.env.MCP_PREFLIGHT_LLM_MODEL.trim()
|
||||||
|
try {
|
||||||
|
const configured = await readOpenShellInferenceModel()
|
||||||
|
if (configured) return configured
|
||||||
|
} catch {
|
||||||
|
// Callers surface endpoint failures; model resolution falls through to explicit env compatibility.
|
||||||
|
}
|
||||||
|
return process.env.OPENAI_MODEL?.trim() || process.env.VLLM_MODEL?.trim() || ""
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveOpenAiCompatibleBaseUrl() {
|
||||||
|
return (process.env.MCP_PREFLIGHT_LLM_BASE_URL || process.env.OPENAI_BASE_URL || process.env.VLLM_BASE_URL || "http://localhost:8000/v1").replace(/\/+$/, "")
|
||||||
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import { readdir, readFile, stat, writeFile } from "node:fs/promises"
|
import { readdir, readFile, stat, writeFile } from "node:fs/promises"
|
||||||
import path from "node:path"
|
import path from "node:path"
|
||||||
|
import { resolveOpenAiCompatibleBaseUrl, resolvePrimaryInferenceModel } from "./inferenceModel"
|
||||||
import { getSandboxInferenceConfig } from "./sandboxInferenceStore"
|
import { getSandboxInferenceConfig } from "./sandboxInferenceStore"
|
||||||
import type { McpPreflightResult } from "./mcpPreflight"
|
import type { McpPreflightResult } from "./mcpPreflight"
|
||||||
import type { McpServerInstall } from "./mcpServerStore"
|
import type { McpServerInstall } from "./mcpServerStore"
|
||||||
@@ -46,12 +47,6 @@ const MAX_CONTEXT_FILE_BYTES = Number.parseInt(process.env.MCP_PREFLIGHT_REPAIR_
|
|||||||
const MAX_CONTEXT_FILES = Number.parseInt(process.env.MCP_PREFLIGHT_REPAIR_MAX_FILES || "18", 10)
|
const MAX_CONTEXT_FILES = Number.parseInt(process.env.MCP_PREFLIGHT_REPAIR_MAX_FILES || "18", 10)
|
||||||
const MAX_REPAIR_FILE_BYTES = Number.parseInt(process.env.MCP_PREFLIGHT_REPAIR_MAX_OUTPUT_FILE_BYTES || String(512 * 1024), 10)
|
const MAX_REPAIR_FILE_BYTES = Number.parseInt(process.env.MCP_PREFLIGHT_REPAIR_MAX_OUTPUT_FILE_BYTES || String(512 * 1024), 10)
|
||||||
const REQUEST_TIMEOUT_MS = Number.parseInt(process.env.MCP_PREFLIGHT_REPAIR_TIMEOUT_MS || "60000", 10)
|
const REQUEST_TIMEOUT_MS = Number.parseInt(process.env.MCP_PREFLIGHT_REPAIR_TIMEOUT_MS || "60000", 10)
|
||||||
const DEFAULT_MODEL = process.env.MCP_PREFLIGHT_LLM_MODEL || process.env.OPENAI_MODEL || "gpt-4o-mini"
|
|
||||||
const DEFAULT_BASE_URL = process.env.MCP_PREFLIGHT_LLM_BASE_URL || process.env.OPENAI_BASE_URL || process.env.VLLM_BASE_URL || "http://localhost:8000/v1"
|
|
||||||
|
|
||||||
function cleanBaseUrl(value: string) {
|
|
||||||
return value.replace(/\/+$/, "")
|
|
||||||
}
|
|
||||||
|
|
||||||
function relativeInside(root: string, candidate: string) {
|
function relativeInside(root: string, candidate: string) {
|
||||||
const fullPath = path.resolve(root, candidate)
|
const fullPath = path.resolve(root, candidate)
|
||||||
@@ -110,20 +105,16 @@ async function readContextFiles(root: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function resolveRepairRoute(sandboxId?: string | null) {
|
async function resolveRepairRoute(sandboxId?: string | null) {
|
||||||
|
let explicitModel = ""
|
||||||
if (sandboxId) {
|
if (sandboxId) {
|
||||||
const config = await getSandboxInferenceConfig(sandboxId)
|
const config = await getSandboxInferenceConfig(sandboxId)
|
||||||
const enabled = config.routes.filter((route) => route.enabled)
|
const enabled = config.routes.filter((route) => route.enabled)
|
||||||
const primary = enabled.find((route) => route.id === config.primaryRouteId) || enabled[0]
|
const primary = enabled.find((route) => route.id === config.primaryRouteId) || enabled[0]
|
||||||
if (primary?.model) {
|
explicitModel = primary?.model || ""
|
||||||
return {
|
|
||||||
model: process.env.MCP_PREFLIGHT_LLM_MODEL || primary.model,
|
|
||||||
baseUrl: cleanBaseUrl(DEFAULT_BASE_URL),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
model: DEFAULT_MODEL,
|
model: await resolvePrimaryInferenceModel(explicitModel),
|
||||||
baseUrl: cleanBaseUrl(DEFAULT_BASE_URL),
|
baseUrl: resolveOpenAiCompatibleBaseUrl(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -137,6 +128,7 @@ function extractJsonObject(value: string) {
|
|||||||
|
|
||||||
async function requestRepairPlan(context: RepairContext, files: Awaited<ReturnType<typeof readContextFiles>>) {
|
async function requestRepairPlan(context: RepairContext, files: Awaited<ReturnType<typeof readContextFiles>>) {
|
||||||
const route = await resolveRepairRoute(context.sandboxId)
|
const route = await resolveRepairRoute(context.sandboxId)
|
||||||
|
if (!route.model) throw new Error("No primary inference model is configured")
|
||||||
const controller = new AbortController()
|
const controller = new AbortController()
|
||||||
const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS)
|
const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS)
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ const registriesRoutePath = path.join(root, 'app/api/mcp/registries/route.ts')
|
|||||||
const registriesAssistRoutePath = path.join(root, 'app/api/mcp/registries/assist/route.ts')
|
const registriesAssistRoutePath = path.join(root, 'app/api/mcp/registries/assist/route.ts')
|
||||||
const installAssistRoutePath = path.join(root, 'app/api/mcp/install-assist/route.ts')
|
const installAssistRoutePath = path.join(root, 'app/api/mcp/install-assist/route.ts')
|
||||||
const registryStorePath = path.join(root, 'app/lib/mcpRegistryStore.ts')
|
const registryStorePath = path.join(root, 'app/lib/mcpRegistryStore.ts')
|
||||||
|
const inferenceModelPath = path.join(root, 'app/lib/inferenceModel.ts')
|
||||||
const brokerCapabilitiesRoutePath = path.join(root, 'app/api/mcp/broker/capabilities/route.ts')
|
const brokerCapabilitiesRoutePath = path.join(root, 'app/api/mcp/broker/capabilities/route.ts')
|
||||||
const brokerCallRoutePath = path.join(root, 'app/api/mcp/broker/call/route.ts')
|
const brokerCallRoutePath = path.join(root, 'app/api/mcp/broker/call/route.ts')
|
||||||
const sandboxMcpRoutePath = path.join(root, 'app/api/sandbox/[sandboxId]/mcp/route.ts')
|
const sandboxMcpRoutePath = path.join(root, 'app/api/sandbox/[sandboxId]/mcp/route.ts')
|
||||||
@@ -31,7 +32,7 @@ const sidebarPath = path.join(root, 'app/components/Sidebar.tsx')
|
|||||||
const sandboxListPath = path.join(root, 'app/components/SandboxList.tsx')
|
const sandboxListPath = path.join(root, 'app/components/SandboxList.tsx')
|
||||||
const pagePath = path.join(root, 'app/page.tsx')
|
const pagePath = path.join(root, 'app/page.tsx')
|
||||||
|
|
||||||
const [storeSource, brokerStoreSource, brokerClientSource, manifestSource, privilegedFilesSource, routeSource, uploadRouteSource, preflightRouteSource, healthRouteSource, registryRouteSource, registriesRouteSource, registriesAssistRouteSource, installAssistRouteSource, registryStoreSource, brokerCapabilitiesRouteSource, brokerCallRouteSource, sandboxMcpRouteSource, brokerUrlSource, preflightLibSource, preflightRepairLibSource, middlewareSource, panelSource, helpSource, sidebarSource, sandboxListSource, pageSource] = await Promise.all([
|
const [storeSource, brokerStoreSource, brokerClientSource, manifestSource, privilegedFilesSource, routeSource, uploadRouteSource, preflightRouteSource, healthRouteSource, registryRouteSource, registriesRouteSource, registriesAssistRouteSource, installAssistRouteSource, registryStoreSource, inferenceModelSource, brokerCapabilitiesRouteSource, brokerCallRouteSource, sandboxMcpRouteSource, brokerUrlSource, preflightLibSource, preflightRepairLibSource, middlewareSource, panelSource, helpSource, sidebarSource, sandboxListSource, pageSource] = await Promise.all([
|
||||||
readFile(storePath, 'utf8'),
|
readFile(storePath, 'utf8'),
|
||||||
readFile(brokerStorePath, 'utf8'),
|
readFile(brokerStorePath, 'utf8'),
|
||||||
readFile(brokerClientPath, 'utf8'),
|
readFile(brokerClientPath, 'utf8'),
|
||||||
@@ -46,6 +47,7 @@ const [storeSource, brokerStoreSource, brokerClientSource, manifestSource, privi
|
|||||||
readFile(registriesAssistRoutePath, 'utf8'),
|
readFile(registriesAssistRoutePath, 'utf8'),
|
||||||
readFile(installAssistRoutePath, 'utf8'),
|
readFile(installAssistRoutePath, 'utf8'),
|
||||||
readFile(registryStorePath, 'utf8'),
|
readFile(registryStorePath, 'utf8'),
|
||||||
|
readFile(inferenceModelPath, 'utf8'),
|
||||||
readFile(brokerCapabilitiesRoutePath, 'utf8'),
|
readFile(brokerCapabilitiesRoutePath, 'utf8'),
|
||||||
readFile(brokerCallRoutePath, 'utf8'),
|
readFile(brokerCallRoutePath, 'utf8'),
|
||||||
readFile(sandboxMcpRoutePath, 'utf8'),
|
readFile(sandboxMcpRoutePath, 'utf8'),
|
||||||
@@ -147,9 +149,15 @@ assert.match(registriesRouteSource, /saveMcpRegistry/, 'MCP registries API must
|
|||||||
assert.match(registriesRouteSource, /deleteMcpRegistry/, 'MCP registries API must delete registries')
|
assert.match(registriesRouteSource, /deleteMcpRegistry/, 'MCP registries API must delete registries')
|
||||||
assert.match(registriesAssistRouteSource, /chat\/completions/, 'MCP registry assistant must use the running OpenAI-compatible LLM endpoint')
|
assert.match(registriesAssistRouteSource, /chat\/completions/, 'MCP registry assistant must use the running OpenAI-compatible LLM endpoint')
|
||||||
assert.match(registriesAssistRouteSource, /response_format: \{ type: "json_object" \}/, 'MCP registry assistant must request structured JSON')
|
assert.match(registriesAssistRouteSource, /response_format: \{ type: "json_object" \}/, 'MCP registry assistant must request structured JSON')
|
||||||
|
assert.match(registriesAssistRouteSource, /resolvePrimaryInferenceModel/, 'MCP registry assistant must use the configured primary inference model')
|
||||||
assert.match(installAssistRouteSource, /chat\/completions/, 'MCP install wizard assistant must use the running OpenAI-compatible LLM endpoint')
|
assert.match(installAssistRouteSource, /chat\/completions/, 'MCP install wizard assistant must use the running OpenAI-compatible LLM endpoint')
|
||||||
assert.match(installAssistRouteSource, /uploadRuntime/, 'MCP install wizard assistant must draft upload metadata')
|
assert.match(installAssistRouteSource, /uploadRuntime/, 'MCP install wizard assistant must draft upload metadata')
|
||||||
assert.match(installAssistRouteSource, /response_format: \{ type: "json_object" \}/, 'MCP install wizard assistant must request structured JSON')
|
assert.match(installAssistRouteSource, /response_format: \{ type: "json_object" \}/, 'MCP install wizard assistant must request structured JSON')
|
||||||
|
assert.match(installAssistRouteSource, /resolvePrimaryInferenceModel/, 'MCP install wizard assistant must use the configured primary inference model')
|
||||||
|
assert.match(inferenceModelSource, /\["inference", "get"\]/, 'MCP LLM assistants must resolve the configured OpenShell inference route')
|
||||||
|
assert.match(inferenceModelSource, /Gateway inference/, 'MCP LLM assistants must prefer gateway inference model resolution')
|
||||||
|
assert.match(inferenceModelSource, /System inference/, 'MCP LLM assistants may fall back to system inference model resolution')
|
||||||
|
assert.doesNotMatch(inferenceModelSource, /gpt-4o-mini/, 'MCP LLM assistants must not hardcode a ChatGPT model fallback')
|
||||||
assert.match(panelSource, /Install Server Command/, 'MCP page must expose a server command install action')
|
assert.match(panelSource, /Install Server Command/, 'MCP page must expose a server command install action')
|
||||||
assert.match(panelSource, /Server Install Wizard/, 'MCP page must present custom installation as a wizard')
|
assert.match(panelSource, /Server Install Wizard/, 'MCP page must present custom installation as a wizard')
|
||||||
assert.match(panelSource, /Upload and Preflight/, 'MCP custom server accordion must support uploaded server bundles')
|
assert.match(panelSource, /Upload and Preflight/, 'MCP custom server accordion must support uploaded server bundles')
|
||||||
|
|||||||
Reference in New Issue
Block a user