37 lines
1.0 KiB
YAML
37 lines
1.0 KiB
YAML
# AIPerf 0.10.0+ Schema 2.0 — Extreme Context Test Template (10k Input / 10k Output)
|
||
# Use case: Stress-test context-window limits and sustained decode performance
|
||
# Concurrency is 1 because runtime is 200–1000 seconds per request
|
||
|
||
schema_version: "2.0"
|
||
benchmark:
|
||
models:
|
||
items:
|
||
- name: "vllm/gemma-4-31b-it"
|
||
endpoint:
|
||
urls: ["https://ai.noris.de/v1"]
|
||
type: "chat"
|
||
streaming: true
|
||
datasets:
|
||
- type: "synthetic"
|
||
name: "main"
|
||
entries: 10
|
||
isl:
|
||
type: "fixed"
|
||
value: 10000
|
||
osl:
|
||
type: "fixed"
|
||
value: 10000
|
||
phases:
|
||
- type: "concurrency"
|
||
name: "profiling"
|
||
requests: 10
|
||
|
||
# CLI usage (per model):
|
||
# aiperf profile --config this-file.yaml --concurrency 1 --ui none \
|
||
# --api-key "$API_KEY" --artifact-dir results/10k/MODEL_NAME/
|
||
|
||
# Expected runtime:
|
||
# At 10–50 tok/s output, one request = 200–1000 seconds
|
||
# 10 requests sequentially = ~30–60 minutes per model
|
||
# Schedule with no-concurrency (--concurrency 1) to avoid timeouts
|