Initial commit: Hermes Agent Skills collection
This commit is contained in:
Debian
@@ -0,0 +1,201 @@
|
||||
# OpenWebUI ↔ ComfyUI Integration
|
||||
|
||||
OpenWebUI's native ComfyUI connector is hard-wired for SD/SDXL pipelines (CLIP-L + T5XXL, `EmptyLatentImage`, standard VAE). It **does not work** with FLUX.2 (Qwen3-VL, `EmptyFlux2LatentImage`, TAE/VAE).
|
||||
|
||||
The cleanest solution is a lightweight OpenAI-compatible adapter that exposes `/v1/images/generations` and internally submits a FLUX.2 workflow to ComfyUI's REST API.
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
OpenWebUI ──POST /v1/images/generations──► Adapter ──POST /api/prompt──► ComfyUI
|
||||
(CT 135) (OpenAI format) (Docker) (workflow JSON) (CT 204)
|
||||
```
|
||||
|
||||
The adapter runs as a Docker container (or Python service) on a host that has Layer-2/Layer-3 reachability to ComfyUI.
|
||||
|
||||
---
|
||||
|
||||
## Adapter Code
|
||||
|
||||
### `adapter.py`
|
||||
|
||||
```python
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
import httpx
|
||||
import json
|
||||
import base64
|
||||
import asyncio
|
||||
import time
|
||||
import random
|
||||
import os
|
||||
|
||||
app = FastAPI(title="ComfyUI OpenAI Adapter", version="0.1.0")
|
||||
|
||||
COMFYUI_URL = os.environ.get("COMFYUI_URL", "http://127.0.0.1:8188")
|
||||
WORKFLOW_PREFIX = os.environ.get("WORKFLOW_PREFIX", "oai_adapter")
|
||||
|
||||
FLUX2_WORKFLOW = {
|
||||
"1": {"inputs": {"unet_name": "flux-2-klein-9b-Q4_K_S.gguf"}, "class_type": "UnetLoaderGGUF"},
|
||||
"2": {"inputs": {"clip_name": "Qwen3VL-8B-Instruct-Q4_K_M.gguf", "type": "flux2"}, "class_type": "CLIPLoaderGGUF"},
|
||||
"3": {"inputs": {"vae_name": "taef2"}, "class_type": "VAELoader"},
|
||||
"4": {"inputs": {"width": 1024, "height": 1024, "batch_size": 1}, "class_type": "EmptyFlux2LatentImage"},
|
||||
"5": {"inputs": {"text": "", "clip": ["2", 0]}, "class_type": "CLIPTextEncode"},
|
||||
"6": {"inputs": {"text": "", "clip": ["2", 0]}, "class_type": "CLIPTextEncode"},
|
||||
"7": {"inputs": {"seed": 42, "steps": 4, "cfg": 1.0, "sampler_name": "euler", "scheduler": "simple", "denoise": 1.0,
|
||||
"model": ["1", 0], "positive": ["5", 0], "negative": ["6", 0], "latent_image": ["4", 0]},
|
||||
"class_type": "KSampler"},
|
||||
"8": {"inputs": {"samples": ["7", 0], "vae": ["3", 0]}, "class_type": "VAEDecode"},
|
||||
"9": {"inputs": {"filename_prefix": WORKFLOW_PREFIX, "images": ["8", 0]}, "class_type": "SaveImage"}
|
||||
}
|
||||
|
||||
class ImageRequest(BaseModel):
|
||||
prompt: str
|
||||
n: Optional[int] = 1
|
||||
size: Optional[str] = "1024x1024"
|
||||
response_format: Optional[str] = "b64_json"
|
||||
model: Optional[str] = "flux2"
|
||||
|
||||
async def queue_workflow(workflow: dict) -> str:
|
||||
async with httpx.AsyncClient() as client:
|
||||
r = await client.post(f"{COMFYUI_URL}/api/prompt", json={"prompt": workflow})
|
||||
r.raise_for_status()
|
||||
return r.json()["prompt_id"]
|
||||
|
||||
async def wait_for_image(prompt_id: str, timeout: int = 300) -> list:
|
||||
async with httpx.AsyncClient() as client:
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
r = await client.get(f"{COMFYUI_URL}/history/{prompt_id}")
|
||||
if r.status_code == 200:
|
||||
data = r.json()
|
||||
if prompt_id in data and "outputs" in data[prompt_id]:
|
||||
outputs = []
|
||||
for node_id, node_out in data[prompt_id]["outputs"].items():
|
||||
if "images" in node_out:
|
||||
outputs.extend(node_out["images"])
|
||||
if outputs:
|
||||
return outputs
|
||||
await asyncio.sleep(1)
|
||||
raise TimeoutError(f"Timeout waiting for prompt {prompt_id}")
|
||||
|
||||
async def fetch_image(filename: str, subfolder: str = "", folder_type: str = "output") -> bytes:
|
||||
async with httpx.AsyncClient() as client:
|
||||
params = {"filename": filename, "subfolder": subfolder, "type": folder_type}
|
||||
r = await client.get(f"{COMFYUI_URL}/view", params=params)
|
||||
r.raise_for_status()
|
||||
return r.content
|
||||
|
||||
@app.post("/v1/images/generations")
|
||||
async def generate_images(req: ImageRequest):
|
||||
if req.n > 4:
|
||||
return JSONResponse({"error": {"message": "n must be <= 4", "type": "invalid_request_error"}}, status_code=400)
|
||||
|
||||
try:
|
||||
w, h = map(int, req.size.split("x"))
|
||||
except ValueError:
|
||||
w, h = 1024, 1024
|
||||
|
||||
# CRITICAL: clamp size to prevent timeout/OOM on consumer GPUs
|
||||
MAX_SIZE = 1536
|
||||
if w > MAX_SIZE or h > MAX_SIZE:
|
||||
scale = MAX_SIZE / max(w, h)
|
||||
w = int(w * scale)
|
||||
h = int(h * scale)
|
||||
|
||||
images = []
|
||||
for _ in range(req.n):
|
||||
wf = json.loads(json.dumps(FLUX2_WORKFLOW))
|
||||
wf["5"]["inputs"]["text"] = req.prompt
|
||||
wf["4"]["inputs"]["width"] = w
|
||||
wf["4"]["inputs"]["height"] = h
|
||||
wf["7"]["inputs"]["seed"] = random.randint(1, 2**32)
|
||||
|
||||
prompt_id = await queue_workflow(wf)
|
||||
imgs = await wait_for_image(prompt_id, timeout=300)
|
||||
if not imgs:
|
||||
raise RuntimeError("No images returned")
|
||||
|
||||
img_data = await fetch_image(imgs[0]["filename"], imgs[0].get("subfolder", ""), imgs[0].get("type", "output"))
|
||||
b64 = base64.b64encode(img_data).decode()
|
||||
images.append({"b64_json": b64})
|
||||
|
||||
return {"created": int(time.time()), "data": images}
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
return {"status": "ok"}
|
||||
```
|
||||
|
||||
### `Dockerfile`
|
||||
|
||||
```dockerfile
|
||||
FROM python:3.12-slim
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY adapter.py .
|
||||
EXPOSE 9000
|
||||
CMD ["uvicorn", "adapter:app", "--host", "0.0.0.0", "--port", "9000"]
|
||||
```
|
||||
|
||||
### `requirements.txt`
|
||||
|
||||
```text
|
||||
fastapi>=0.110.0
|
||||
uvicorn>=0.29.0
|
||||
httpx>=0.27.0
|
||||
pydantic>=2.0.0
|
||||
```
|
||||
|
||||
### `docker-compose.yml`
|
||||
|
||||
```yaml
|
||||
services:
|
||||
comfyui-openai-adapter:
|
||||
build: .
|
||||
container_name: comfyui-openai-adapter
|
||||
ports:
|
||||
- "9000:9000"
|
||||
environment:
|
||||
- COMFYUI_URL=http://10.0.30.97:8188 # <-- adapt to your ComfyUI IP
|
||||
- WORKFLOW_PREFIX=oai_bifrost
|
||||
restart: unless-stopped
|
||||
```
|
||||
|
||||
> **Network tip:** If the adapter and ComfyUI are on the same Layer-2 segment, prefer `--network host` (or attach the container to an existing bridge that reaches ComfyUI) to avoid NAT/port-mapping issues.
|
||||
|
||||
---
|
||||
|
||||
## OpenWebUI Configuration
|
||||
|
||||
1. Admin Panel → **Settings** → **Images**
|
||||
2. **Image Generation Engine:** `Open AI`
|
||||
3. **OpenAI API Base URL:** `http://<adapter-host>:9000/v1`
|
||||
4. **OpenAI API Key:** any dummy string (the adapter does not validate keys)
|
||||
5. **Model:** `flux2` (or leave default)
|
||||
|
||||
---
|
||||
|
||||
## Critical Pitfalls
|
||||
|
||||
| Issue | Cause | Fix |
|
||||
|---|---|---|
|
||||
| `TimeoutError` after 300 s | OpenWebUI/DALL-E 3 sends `size: "4096x4096"` | Adapter clamps to `MAX_SIZE = 1536` |
|
||||
| OOM / CUDA out of memory | Resolution too high for 48 GB shared VRAM | Keep ≤ 1536 px on consumer APUs |
|
||||
| `500 Internal Server Error` | ComfyUI node missing (`CLIPLoaderGGUF`, etc.) | Run `comfy node install comfyui-gguf` and verify with `curl /object_info` |
|
||||
| Adapter cannot reach ComfyUI | Docker bridge isolation | Use `--network host` or place both on same bridge |
|
||||
| Empty/black images | Wrong VAE (`ae.safetensors` instead of `taef2` for FLUX.2) | Use `taef2` TAE or the FLUX.2-specific VAE |
|
||||
|
||||
---
|
||||
|
||||
## Environment Snapshot (Schön Consulting, 2026-06-20)
|
||||
|
||||
- **ComfyUI:** PVE CT 204, `10.0.30.97:8188`, ROCm gfx1150 (Radeon 890M)
|
||||
- **OpenWebUI:** PVE CT 135, `10.0.30.102:8080`
|
||||
- **Adapter:** Docker-Host `10.0.30.99:9000` (Portainer node)
|
||||
- **Performance:** 1024×1024 ≈ 110 s, 1536×1536 ≈ 180 s (4 steps, euler/simple)
|
||||
Reference in New Issue
Block a user