# OpenWebUI ↔ ComfyUI Integration OpenWebUI's native ComfyUI connector is hard-wired for SD/SDXL pipelines (CLIP-L + T5XXL, `EmptyLatentImage`, standard VAE). It **does not work** with FLUX.2 (Qwen3-VL, `EmptyFlux2LatentImage`, TAE/VAE). The cleanest solution is a lightweight OpenAI-compatible adapter that exposes `/v1/images/generations` and internally submits a FLUX.2 workflow to ComfyUI's REST API. --- ## Architecture ``` OpenWebUI ──POST /v1/images/generations──► Adapter ──POST /api/prompt──► ComfyUI (CT 135) (OpenAI format) (Docker) (workflow JSON) (CT 204) ``` The adapter runs as a Docker container (or Python service) on a host that has Layer-2/Layer-3 reachability to ComfyUI. --- ## Adapter Code ### `adapter.py` ```python from fastapi import FastAPI from fastapi.responses import JSONResponse from pydantic import BaseModel from typing import Optional import httpx import json import base64 import asyncio import time import random import os app = FastAPI(title="ComfyUI OpenAI Adapter", version="0.1.0") COMFYUI_URL = os.environ.get("COMFYUI_URL", "http://127.0.0.1:8188") WORKFLOW_PREFIX = os.environ.get("WORKFLOW_PREFIX", "oai_adapter") FLUX2_WORKFLOW = { "1": {"inputs": {"unet_name": "flux-2-klein-9b-Q4_K_S.gguf"}, "class_type": "UnetLoaderGGUF"}, "2": {"inputs": {"clip_name": "Qwen3VL-8B-Instruct-Q4_K_M.gguf", "type": "flux2"}, "class_type": "CLIPLoaderGGUF"}, "3": {"inputs": {"vae_name": "taef2"}, "class_type": "VAELoader"}, "4": {"inputs": {"width": 1024, "height": 1024, "batch_size": 1}, "class_type": "EmptyFlux2LatentImage"}, "5": {"inputs": {"text": "", "clip": ["2", 0]}, "class_type": "CLIPTextEncode"}, "6": {"inputs": {"text": "", "clip": ["2", 0]}, "class_type": "CLIPTextEncode"}, "7": {"inputs": {"seed": 42, "steps": 4, "cfg": 1.0, "sampler_name": "euler", "scheduler": "simple", "denoise": 1.0, "model": ["1", 0], "positive": ["5", 0], "negative": ["6", 0], "latent_image": ["4", 0]}, "class_type": "KSampler"}, "8": {"inputs": {"samples": ["7", 0], "vae": ["3", 0]}, "class_type": "VAEDecode"}, "9": {"inputs": {"filename_prefix": WORKFLOW_PREFIX, "images": ["8", 0]}, "class_type": "SaveImage"} } class ImageRequest(BaseModel): prompt: str n: Optional[int] = 1 size: Optional[str] = "1024x1024" response_format: Optional[str] = "b64_json" model: Optional[str] = "flux2" async def queue_workflow(workflow: dict) -> str: async with httpx.AsyncClient() as client: r = await client.post(f"{COMFYUI_URL}/api/prompt", json={"prompt": workflow}) r.raise_for_status() return r.json()["prompt_id"] async def wait_for_image(prompt_id: str, timeout: int = 300) -> list: async with httpx.AsyncClient() as client: start = time.time() while time.time() - start < timeout: r = await client.get(f"{COMFYUI_URL}/history/{prompt_id}") if r.status_code == 200: data = r.json() if prompt_id in data and "outputs" in data[prompt_id]: outputs = [] for node_id, node_out in data[prompt_id]["outputs"].items(): if "images" in node_out: outputs.extend(node_out["images"]) if outputs: return outputs await asyncio.sleep(1) raise TimeoutError(f"Timeout waiting for prompt {prompt_id}") async def fetch_image(filename: str, subfolder: str = "", folder_type: str = "output") -> bytes: async with httpx.AsyncClient() as client: params = {"filename": filename, "subfolder": subfolder, "type": folder_type} r = await client.get(f"{COMFYUI_URL}/view", params=params) r.raise_for_status() return r.content @app.post("/v1/images/generations") async def generate_images(req: ImageRequest): if req.n > 4: return JSONResponse({"error": {"message": "n must be <= 4", "type": "invalid_request_error"}}, status_code=400) try: w, h = map(int, req.size.split("x")) except ValueError: w, h = 1024, 1024 # CRITICAL: clamp size to prevent timeout/OOM on consumer GPUs MAX_SIZE = 1536 if w > MAX_SIZE or h > MAX_SIZE: scale = MAX_SIZE / max(w, h) w = int(w * scale) h = int(h * scale) images = [] for _ in range(req.n): wf = json.loads(json.dumps(FLUX2_WORKFLOW)) wf["5"]["inputs"]["text"] = req.prompt wf["4"]["inputs"]["width"] = w wf["4"]["inputs"]["height"] = h wf["7"]["inputs"]["seed"] = random.randint(1, 2**32) prompt_id = await queue_workflow(wf) imgs = await wait_for_image(prompt_id, timeout=300) if not imgs: raise RuntimeError("No images returned") img_data = await fetch_image(imgs[0]["filename"], imgs[0].get("subfolder", ""), imgs[0].get("type", "output")) b64 = base64.b64encode(img_data).decode() images.append({"b64_json": b64}) return {"created": int(time.time()), "data": images} @app.get("/health") async def health(): return {"status": "ok"} ``` ### `Dockerfile` ```dockerfile FROM python:3.12-slim WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY adapter.py . EXPOSE 9000 CMD ["uvicorn", "adapter:app", "--host", "0.0.0.0", "--port", "9000"] ``` ### `requirements.txt` ```text fastapi>=0.110.0 uvicorn>=0.29.0 httpx>=0.27.0 pydantic>=2.0.0 ``` ### `docker-compose.yml` ```yaml services: comfyui-openai-adapter: build: . container_name: comfyui-openai-adapter ports: - "9000:9000" environment: - COMFYUI_URL=http://10.0.30.97:8188 # <-- adapt to your ComfyUI IP - WORKFLOW_PREFIX=oai_bifrost restart: unless-stopped ``` > **Network tip:** If the adapter and ComfyUI are on the same Layer-2 segment, prefer `--network host` (or attach the container to an existing bridge that reaches ComfyUI) to avoid NAT/port-mapping issues. --- ## OpenWebUI Configuration 1. Admin Panel → **Settings** → **Images** 2. **Image Generation Engine:** `Open AI` 3. **OpenAI API Base URL:** `http://:9000/v1` 4. **OpenAI API Key:** any dummy string (the adapter does not validate keys) 5. **Model:** `flux2` (or leave default) --- ## Critical Pitfalls | Issue | Cause | Fix | |---|---|---| | `TimeoutError` after 300 s | OpenWebUI/DALL-E 3 sends `size: "4096x4096"` | Adapter clamps to `MAX_SIZE = 1536` | | OOM / CUDA out of memory | Resolution too high for 48 GB shared VRAM | Keep ≤ 1536 px on consumer APUs | | `500 Internal Server Error` | ComfyUI node missing (`CLIPLoaderGGUF`, etc.) | Run `comfy node install comfyui-gguf` and verify with `curl /object_info` | | Adapter cannot reach ComfyUI | Docker bridge isolation | Use `--network host` or place both on same bridge | | Empty/black images | Wrong VAE (`ae.safetensors` instead of `taef2` for FLUX.2) | Use `taef2` TAE or the FLUX.2-specific VAE | --- ## Environment Snapshot (Schön Consulting, 2026-06-20) - **ComfyUI:** PVE CT 204, `10.0.30.97:8188`, ROCm gfx1150 (Radeon 890M) - **OpenWebUI:** PVE CT 135, `10.0.30.102:8080` - **Adapter:** Docker-Host `10.0.30.99:9000` (Portainer node) - **Performance:** 1024×1024 ≈ 110 s, 1536×1536 ≈ 180 s (4 steps, euler/simple)