93 lines
7.0 KiB
INI
93 lines
7.0 KiB
INI
# vLLM presets for fl — AMD RX 7600 XT · 16 GB · docker image vllm/vllm-openai-rocm
|
||
# Same convention as ../llamacpp/config.ini: section name = served model id, `key = value` = `--key value`.
|
||
#
|
||
# ⚠ ONE ENGINE AT A TIME. vLLM pre-allocates `gpu-memory-utilization` × 16 GiB at startup and KEEPS it.
|
||
# Any llama.cpp load on top (a client hitting :11343 is enough) overcommits → GTT spill → PC freeze.
|
||
# `vllmctl.py up` refuses to start while llama.service is active — stop the router first
|
||
# (`vllmctl.py up <preset> --stop-router`), start it again after `vllmctl.py down`.
|
||
#
|
||
# Sections:
|
||
# [global] container/runtime knobs (NOT vllm flags) — see vllmctl.py
|
||
# [<preset>] one `vllmctl.py up <preset>`:
|
||
# model HF repo id, or /models/<dir> (models-dir is mounted at /models, read-only)
|
||
# any other key → `--key value` for `vllm serve`; `true` → bare `--key`; `false` → omitted
|
||
# served-model-name defaults to the section name (what clients and bench-vs.py use)
|
||
# x-* vllmctl-only metadata (x-weights-gb feeds the VRAM projection)
|
||
# env.NAME = value container environment (per preset; [global] env.* apply to all)
|
||
#
|
||
# VRAM MATH (vllmctl prints it before every start, aborts under reserve-gb unless --force):
|
||
# budget = gpu-memory-utilization × 17.16 GB (vLLM's total = the card's total, same as sysfs)
|
||
# budget must hold weights + activations + KV cache; everything above it + the desktop must stay
|
||
# under 16 GB with ≥ reserve-gb free. 0.75 → 12.9 GB budget → ~2.9 GB free on a 1.3 GB clean desktop.
|
||
# A busy evening desktop (3+ GB) does NOT fit any of the real models below — clean desktop only.
|
||
# Sizes measured 2026-10-21 from the HF API: the 4-bit vLLM checkpoints are 10-11.5 GB because
|
||
# embeddings, lm_head and the vision tower stay bf16 — bigger than the Q6/Q8 GGUFs they pair with.
|
||
|
||
[global]
|
||
image = vllm/vllm-openai-rocm:v0.32.0 # = `latest` on 2026-10-18 (ROCm 10.0, 11.6 GB download); update = bump this + `vllmctl.py pull`
|
||
container = vllm
|
||
port = 11348 # whisper = 11345/11346, kokoro = 11347
|
||
models-dir = /home/anon/software/models # mounted at /models (ro) — share the GGUF dir
|
||
hf-cache = /home/anon/.cache/huggingface # mounted at /root/.cache/huggingface (downloads + token)
|
||
router = http://127.0.0.1:11343 # llama.cpp router root (for the resident-model gate)
|
||
reserve-gb = 2.0 # abort if projected free VRAM < this (bench.py floor 1.5, target 2.5)
|
||
drain-gb = 4.0 # "card is clean" threshold before starting (same as bench.py)
|
||
health-timeout = 1800 # s — first start compiles Triton kernels, be patient
|
||
gpu-memory-utilization = 0.75 # default; presets override
|
||
# REQUIRED on this card (✓ verified 2026-10-21): v0.32.0 ships no gfx1102 code objects — without the
|
||
# override every GPU kernel launch aborts the process SILENTLY (the engine core just vanishes, the
|
||
# API server reports "Failed core proc(s): {}"). gfx1100 (RX 7900) code runs on gfx1102, same RDNA3 ISA.
|
||
env.HSA_OVERRIDE_GFX_VERSION = 11.0.0
|
||
# Attention backend knobs (ROCm; names moved between versions — `vllmctl.py exec vllm serve --help`):
|
||
# env.VLLM_ATTENTION_BACKEND = TRITON_ATTN # the default on RDNA; set explicitly if a hang points at attention
|
||
|
||
# ─── 1. smoke test — tiny, certain to be supported; proves docker + GPU + image + override ────
|
||
# ✓ 2026-10-21: 154.9 t/s decode · 1462 t/s prefill · TTFT 0.02 s · 7.7 GB used incl. a 2.9 GB desktop.
|
||
[Qwen3-0.6B-smoke]
|
||
model = Qwen/Qwen3-0.6B
|
||
gpu-memory-utilization = 0.30
|
||
max-model-len = 4096
|
||
x-weights-gb = 1.5
|
||
|
||
# ─── 2. gemma-4-E4B — ✗ DOES NOT FIT vLLM ON THIS CARD (2026-10-21), tombstone kept for the math ──
|
||
# Pairs with llama.cpp [gemma-4-E4B-it-UD-Q8_K_XL] (57 t/s with MTP, 96k ctx). google/gemma-4-E4B-it-qat-w4a16-ct
|
||
# is 11.5 GB, and the second try (text-only, batch 2048, @0.73 on a 2.1 GB desktop) ended with
|
||
# "Available KV cache memory: -4.44 GiB" after an OOM on a 5,637,144,576-byte allocation. That is exactly
|
||
# Gemma 4's per-layer embedding table: 262144 vocab × 42 layers × 256 × 2 bytes (bf16). vLLM holds it
|
||
# in bf16 AND needs a second buffer of the same size → ~16 GiB before any KV block. No util, batch or
|
||
# desktop fixes that; the GPTQ/AWQ alternatives carry the same bf16 table. llama.cpp fits because the
|
||
# GGUF stores the table quantized, once. A different image changes nothing here.
|
||
# [gemma-4-E4B-it-w4a16]
|
||
# model = google/gemma-4-E4B-it-qat-w4a16-ct
|
||
# gpu-memory-utilization = 0.78
|
||
# max-model-len = 16384
|
||
# max-num-seqs = 4
|
||
# max-num-batched-tokens = 2048
|
||
# limit-mm-per-prompt = {"image": 0, "audio": 0, "video": 0}
|
||
# x-weights-gb = 11.5
|
||
|
||
# ─── 3. Qwen3.5-9B — pairs with llama.cpp [Qwen3.5-9B-UD-Q6_K_XL] (32 t/s with MTP, 32k ctx) ──
|
||
# Red Hat's llm-compressor int4 (compressed-tensors). 10.95 GB + a 0.49 GB MTP head file.
|
||
# Alternative (real AWQ, Triton kernels on ROCm): QuantTrio/Qwen3.5-9B-AWQ — 12.4 GB, too big for 0.78.
|
||
# The last viable "same model" pair: no per-layer table, but a 248k vocab → the fp32 logits buffer
|
||
# during profiling is max-num-batched-tokens × 248k × 4 B (2048 → 2.0 GB, 512 → 0.5 GB). Start small.
|
||
# On a 2.1 GB desktop the projection allows 0.75 (`--set gpu-memory-utilization=0.75`) → ~1.9 GB for
|
||
# activations + graphs + KV. If "No available memory for the cache blocks" again: `--set enforce-eager=true`
|
||
# (drops the CUDA-graph memory, costs decode speed), then a cleaner desktop for 0.78.
|
||
# ✓ RUNS 2026-10-21 @0.80 (0.75 was 0.11 GiB short of KV room): 29.4 t/s decode · 181 prefill · TTFT 0.17 ·
|
||
# decode@8k 16.0 · 4-stream aggregate 60.6 · 12.5 GB used on a 0.6 GB desktop. Same evening, llama.cpp
|
||
# [Qwen3.5-9B-UD-Q6_K_XL] with its MTP draft: 61.3 · 114 · 0.27 · 51.1 · 85.4 · 10.9 GB. See README.
|
||
[Qwen3.5-9B-w4a16]
|
||
model = RedHatAI/Qwen3.5-9B-quantized.w4a16
|
||
gpu-memory-utilization = 0.80 # needs a ≤ 0.6 GB desktop (projection: 2.9 GB free); 0.75 fails on KV room
|
||
max-model-len = 16384
|
||
max-num-seqs = 4
|
||
max-num-batched-tokens = 512 # caps the profiling logits buffer (prefill t/s cost — note it in the ledger)
|
||
limit-mm-per-prompt = {"image": 0, "video": 0} # text-only bench (Qwen3.5 has no audio input)
|
||
x-weights-gb = 11.0
|
||
# NEXT (the one knob that could change the verdict): MTP speculative decoding — the repo ships
|
||
# model_mtp.safetensors and llama.cpp's 2× comes from exactly this. Costs ~0.5 GB weights + draft
|
||
# state → try @0.82 on a clean desktop. Syntax per `vllmctl.py exec vllm serve --help`:
|
||
# ./vllmctl.py up Qwen3.5-9B-w4a16 --set gpu-memory-utilization=0.82 --set 'speculative-config={"method": "mtp", "num_speculative_tokens": 2}'
|
||
# speculative-config = {"method": "mtp", "num_speculative_tokens": 2}
|