Initial commit: Hermes Agent Skills collection
This commit is contained in:
Debian
@@ -0,0 +1,67 @@
|
||||
# AIPerf 3-Scenario Suite Template
|
||||
|
||||
Reusable structure for benchmarking N models with Sales + Capacity + Steady-State.
|
||||
|
||||
## Directory Layout
|
||||
|
||||
```
|
||||
aiperf-benchmark-suite/
|
||||
├── configs/
|
||||
│ └── <model-safe-name>/
|
||||
│ ├── sales.yaml
|
||||
│ ├── capacity.yaml
|
||||
│ └── steady.yaml
|
||||
├── templates/
|
||||
│ └── base-config.yaml
|
||||
├── scripts/
|
||||
│ ├── run-suite.sh
|
||||
│ └── generate-summary.py
|
||||
├── results/ # gitignored
|
||||
└── README.md
|
||||
```
|
||||
|
||||
## Base Config Template
|
||||
|
||||
```yaml
|
||||
url: "https://<endpoint>/v1"
|
||||
endpoint_type: "chat"
|
||||
extra-headers:
|
||||
Authorization: "Bearer ${API_KEY}"
|
||||
streaming: true
|
||||
tokenizer: "builtin"
|
||||
random_seed: 42
|
||||
```
|
||||
|
||||
## Scenario Definitions
|
||||
|
||||
### Sales
|
||||
- Concurrency: 1, 5, 10 (sequentially)
|
||||
- Request count: 50 per level
|
||||
- Dataset: `sharegpt`
|
||||
- Display: `simple` or `dashboard`
|
||||
- Flags: `--concurrency N --request-count 50 --dataset sharegpt`
|
||||
|
||||
### Capacity
|
||||
- Concurrency: 10, 25, 50, 100 (stepped ramp)
|
||||
- Request count: 100 per level
|
||||
- Dataset: synthetic fixed lengths
|
||||
- Flags: `--concurrency N --request-count 100 --input-tokens-mean 512 --output-tokens-mean 128`
|
||||
|
||||
### Steady-State
|
||||
- Duration: 600 seconds
|
||||
- Concurrency: sweet spot from Capacity (e.g., 20)
|
||||
- Dataset: mixed or `sharegpt`
|
||||
- Flags: `--concurrency 20 --time 600 --dataset sharegpt`
|
||||
|
||||
## Orchestrator Pattern
|
||||
|
||||
Iterate models × scenarios, with `--output $RESULTS_DIR/$MODEL/$SCENARIO/conc${CONC}`.
|
||||
Pause 30s between major runs to avoid rate limits.
|
||||
|
||||
## Result Summary
|
||||
|
||||
After all runs, parse `profile_export_aiperf.json` files to generate markdown table:
|
||||
|
||||
| Model | Scenario | Concurrency | TTFT avg | TTFT p99 | Throughput |
|
||||
|
||||
See `scripts/generate-summary.py` for a reference implementation.
|
||||
@@ -0,0 +1,36 @@
|
||||
# AIPerf 0.10.0+ Schema 2.0 — Extreme Context Test Template (10k Input / 10k Output)
|
||||
# Use case: Stress-test context-window limits and sustained decode performance
|
||||
# Concurrency is 1 because runtime is 200–1000 seconds per request
|
||||
|
||||
schema_version: "2.0"
|
||||
benchmark:
|
||||
models:
|
||||
items:
|
||||
- name: "vllm/gemma-4-31b-it"
|
||||
endpoint:
|
||||
urls: ["https://ai.noris.de/v1"]
|
||||
type: "chat"
|
||||
streaming: true
|
||||
datasets:
|
||||
- type: "synthetic"
|
||||
name: "main"
|
||||
entries: 10
|
||||
isl:
|
||||
type: "fixed"
|
||||
value: 10000
|
||||
osl:
|
||||
type: "fixed"
|
||||
value: 10000
|
||||
phases:
|
||||
- type: "concurrency"
|
||||
name: "profiling"
|
||||
requests: 10
|
||||
|
||||
# CLI usage (per model):
|
||||
# aiperf profile --config this-file.yaml --concurrency 1 --ui none \
|
||||
# --api-key "$API_KEY" --artifact-dir results/10k/MODEL_NAME/
|
||||
|
||||
# Expected runtime:
|
||||
# At 10–50 tok/s output, one request = 200–1000 seconds
|
||||
# 10 requests sequentially = ~30–60 minutes per model
|
||||
# Schedule with no-concurrency (--concurrency 1) to avoid timeouts
|
||||
@@ -0,0 +1,37 @@
|
||||
# AIPerf 0.10.0+ Schema 2.0 — Long-Input Prefill Test Template
|
||||
# Use case: Isolate memory-bandwidth prefill bottleneck for RAG / long-context workloads
|
||||
# Concurrency is kept low (10) to avoid queueing artifacts
|
||||
|
||||
schema_version: "2.0"
|
||||
benchmark:
|
||||
models:
|
||||
items:
|
||||
- name: "moonshotai/kimi-k2.6"
|
||||
endpoint:
|
||||
urls: ["https://ai.noris.de/v1"]
|
||||
type: "chat"
|
||||
streaming: true
|
||||
datasets:
|
||||
- name: "synthetic_long"
|
||||
type: "synthetic"
|
||||
prompt_template:
|
||||
content: "Analyze the following topic in depth and provide detailed insights."
|
||||
strategy:
|
||||
type: "fixed"
|
||||
max_total_tokens: 1152
|
||||
input_tokens: 1024 # Double the standard 512
|
||||
output_tokens: 128 # Short output to isolate prefill
|
||||
phases:
|
||||
- type: "concurrency"
|
||||
name: "profiling"
|
||||
duration: 300 # 5 minutes to amortize cold-start
|
||||
# Do NOT specify requests — duration drives the phase
|
||||
|
||||
# CLI usage:
|
||||
# aiperf profile --config this-file.yaml --concurrency 10 --ui none \
|
||||
# --api-key "$API_KEY" --artifact-dir results/longinput/
|
||||
|
||||
# YAML rules learned the hard way:
|
||||
# - datasets is a LIST (starts with "- name:", NOT "default:")
|
||||
# - prompt_template and strategy are siblings to name/type
|
||||
# - Never use ${VAR} in headers when passing --api-key via CLI
|
||||
@@ -0,0 +1,39 @@
|
||||
schema_version: "2.0"
|
||||
benchmark:
|
||||
models:
|
||||
items:
|
||||
- name: "vllm/MODEL-NAME-HERE"
|
||||
strategy: "round_robin"
|
||||
endpoint:
|
||||
urls:
|
||||
- "https://ENDPOINT/v1"
|
||||
type: "chat"
|
||||
streaming: true
|
||||
headers:
|
||||
Authorization: "Bearer ${API_KEY}"
|
||||
# --- Option A: Public dataset (ShareGPT) ---
|
||||
datasets:
|
||||
- type: "public"
|
||||
name: "main"
|
||||
dataset: "sharegpt"
|
||||
sampling: "shuffle"
|
||||
# --- Option B: Synthetic dataset (no download) ---
|
||||
# datasets:
|
||||
# - type: "synthetic"
|
||||
# name: "main"
|
||||
# entries: 100
|
||||
# isl:
|
||||
# type: "fixed"
|
||||
# value: 512
|
||||
# osl:
|
||||
# type: "fixed"
|
||||
# value: 128
|
||||
phases:
|
||||
- type: "concurrency"
|
||||
name: "profiling"
|
||||
# Choose ONE: requests OR duration
|
||||
requests: 50 # for count-based (sales, capacity)
|
||||
sessions: 1 # concurrency level (overridden by --concurrency CLI flag)
|
||||
# duration: 600 # for time-based (steady-state)
|
||||
tokenizer:
|
||||
name: "builtin"
|
||||
@@ -0,0 +1,139 @@
|
||||
# AIPerf 3-Scenario Benchmark Suite Template
|
||||
# Copy and customize per target endpoint.
|
||||
# Usage: aiperf profile -c <this-file>
|
||||
#
|
||||
# Environment variables expected:
|
||||
# API_KEY - Bearer token for the target endpoint
|
||||
# API_BASE_URL - e.g. https://ai.noris.de
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# SCENARIO 1: Sales (Representative Load)
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
scenarios:
|
||||
- name: sales-low
|
||||
model: vllm/moonshotai/kimi-k2.6
|
||||
url: "${API_BASE_URL}"
|
||||
endpoint-type: chat
|
||||
streaming: true
|
||||
tokenizer: moonshotai/kimi-k2.6
|
||||
concurrency: 1
|
||||
request-count: 50
|
||||
dataset: sharegpt
|
||||
extra-headers:
|
||||
Authorization: "Bearer ${API_KEY}"
|
||||
output-directory: "results/${DATE}/sales/concurrency-1"
|
||||
ui: simple
|
||||
|
||||
- name: sales-moderate
|
||||
model: vllm/moonshotai/kimi-k2.6
|
||||
url: "${API_BASE_URL}"
|
||||
endpoint-type: chat
|
||||
streaming: true
|
||||
tokenizer: moonshotai/kimi-k2.6
|
||||
concurrency: 5
|
||||
request-count: 50
|
||||
dataset: sharegpt
|
||||
extra-headers:
|
||||
Authorization: "Bearer ${API_KEY}"
|
||||
output-directory: "results/${DATE}/sales/concurrency-5"
|
||||
ui: simple
|
||||
|
||||
- name: sales-realistic
|
||||
model: vllm/moonshotai/kimi-k2.6
|
||||
url: "${API_BASE_URL}"
|
||||
endpoint-type: chat
|
||||
streaming: true
|
||||
tokenizer: moonshotai/kimi-k2.6
|
||||
concurrency: 10
|
||||
request-count: 50
|
||||
dataset: sharegpt
|
||||
extra-headers:
|
||||
Authorization: "Bearer ${API_KEY}"
|
||||
output-directory: "results/${DATE}/sales/concurrency-10"
|
||||
ui: simple
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# SCENARIO 2: Capacity (Breakpoint Discovery)
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
- name: capacity-10
|
||||
model: vllm/moonshotai/kimi-k2.6
|
||||
url: "${API_BASE_URL}"
|
||||
endpoint-type: chat
|
||||
streaming: true
|
||||
tokenizer: moonshotai/kimi-k2.6
|
||||
concurrency: 10
|
||||
request-count: 100
|
||||
dataset: synthetic
|
||||
input-tokens-mean: 512
|
||||
output-tokens-mean: 128
|
||||
extra-headers:
|
||||
Authorization: "Bearer ${API_KEY}"
|
||||
output-directory: "results/${DATE}/capacity/concurrency-10"
|
||||
ui: none
|
||||
|
||||
- name: capacity-25
|
||||
model: vllm/moonshotai/kimi-k2.6
|
||||
url: "${API_BASE_URL}"
|
||||
endpoint-type: chat
|
||||
streaming: true
|
||||
tokenizer: moonshotai/kimi-k2.6
|
||||
concurrency: 25
|
||||
request-count: 100
|
||||
dataset: synthetic
|
||||
input-tokens-mean: 512
|
||||
output-tokens-mean: 128
|
||||
extra-headers:
|
||||
Authorization: "Bearer ${API_KEY}"
|
||||
output-directory: "results/${DATE}/capacity/concurrency-25"
|
||||
ui: none
|
||||
|
||||
- name: capacity-50
|
||||
model: vllm/moonshotai/kimi-k2.6
|
||||
url: "${API_BASE_URL}"
|
||||
endpoint-type: chat
|
||||
streaming: true
|
||||
tokenizer: moonshotai/kimi-k2.6
|
||||
concurrency: 50
|
||||
request-count: 100
|
||||
dataset: synthetic
|
||||
input-tokens-mean: 512
|
||||
output-tokens-mean: 128
|
||||
extra-headers:
|
||||
Authorization: "Bearer ${API_KEY}"
|
||||
output-directory: "results/${DATE}/capacity/concurrency-50"
|
||||
ui: none
|
||||
|
||||
- name: capacity-100
|
||||
model: vllm/moonshotai/kimi-k2.6
|
||||
url: "${API_BASE_URL}"
|
||||
endpoint-type: chat
|
||||
streaming: true
|
||||
tokenizer: moonshotai/kimi-k2.6
|
||||
concurrency: 100
|
||||
request-count: 100
|
||||
dataset: synthetic
|
||||
input-tokens-mean: 512
|
||||
output-tokens-mean: 128
|
||||
extra-headers:
|
||||
Authorization: "Bearer ${API_KEY}"
|
||||
output-directory: "results/${DATE}/capacity/concurrency-100"
|
||||
ui: none
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# SCENARIO 3: Steady-State (Long-Term Stability)
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
- name: steady-state
|
||||
model: vllm/moonshotai/kimi-k2.6
|
||||
url: "${API_BASE_URL}"
|
||||
endpoint-type: chat
|
||||
streaming: true
|
||||
tokenizer: moonshotai/kimi-k2.6
|
||||
concurrency: 20
|
||||
duration: 600
|
||||
dataset: sharegpt
|
||||
extra-headers:
|
||||
Authorization: "Bearer ${API_KEY}"
|
||||
output-directory: "results/${DATE}/steady-state"
|
||||
ui: none
|
||||
# Enable timeslice metrics for drift detection:
|
||||
# timeslice: 60
|
||||
@@ -0,0 +1,37 @@
|
||||
schema_version: "2.0"
|
||||
benchmark:
|
||||
models:
|
||||
items:
|
||||
- name: "vllm/REPLACE-MODEL-NAME"
|
||||
endpoint:
|
||||
urls: ["https://ENDPOINT/v1"]
|
||||
type: "chat"
|
||||
streaming: true
|
||||
datasets:
|
||||
default:
|
||||
- name: "synthetic_long"
|
||||
type: "synthetic"
|
||||
prompt_template:
|
||||
content: "Summarize the following text concisely: {{text}}"
|
||||
variables:
|
||||
- name: "text"
|
||||
values: [
|
||||
"artificial intelligence",
|
||||
"quantum computing",
|
||||
"blockchain technology",
|
||||
"climate change",
|
||||
"machine learning",
|
||||
"renewable energy"
|
||||
]
|
||||
strategy:
|
||||
type: "fixed"
|
||||
max_total_tokens: 1152 # input + output + padding
|
||||
input_tokens: 1024 # TARGET INPUT LENGTH: sweep 512 / 1024 / 2048 / 4096
|
||||
output_tokens: 128
|
||||
phases:
|
||||
- type: "concurrency"
|
||||
name: "prefill_stress"
|
||||
duration: 300 # 5 minutes
|
||||
sessions: 10 # fixed low concurrency
|
||||
tokenizer:
|
||||
name: "builtin"
|
||||
Reference in New Issue
Block a user