# vLLM presets for fl — AMD RX 7600 XT · 16 GB · docker image vllm/vllm-openai-rocm
# Same convention as ../llamacpp/config.ini: section name = served model id, `key = value` = `--key value`.
#
# ⚠ ONE ENGINE AT A TIME. vLLM pre-allocates `gpu-memory-utilization` × 16 GiB at startup and KEEPS it.
#    Any llama.cpp load on top (a client hitting :11343 is enough) overcommits → GTT spill → PC freeze.
#    `vllmctl.py up` refuses to start while llama.service is active — stop the router first
#    (`vllmctl.py up <preset> --stop-router`), start it again after `vllmctl.py down`.
#
# Sections:
#   [global]      container/runtime knobs (NOT vllm flags) — see vllmctl.py
#   [<preset>]    one `vllmctl.py up <preset>`:
#       model                  HF repo id, or /models/<dir> (models-dir is mounted at /models, read-only)
#       any other key          → `--key value` for `vllm serve`; `true` → bare `--key`; `false` → omitted
#       served-model-name      defaults to the section name (what clients and bench-vs.py use)
#       x-*                    vllmctl-only metadata (x-weights-gb feeds the VRAM projection)
#       env.NAME = value       container environment (per preset; [global] env.* apply to all)
#
# VRAM MATH (vllmctl prints it before every start, aborts under reserve-gb unless --force):
#   budget = gpu-memory-utilization × 17.16 GB   (vLLM's total = the card's total, same as sysfs)
#   budget must hold weights + activations + KV cache; everything above it + the desktop must stay
#   under 16 GB with ≥ reserve-gb free. 0.75 → 12.9 GB budget → ~2.9 GB free on a 1.3 GB clean desktop.
#   A busy evening desktop (3+ GB) does NOT fit any of the real models below — clean desktop only.
# Sizes measured 2026-10-21 from the HF API: the 4-bit vLLM checkpoints are 10-11.5 GB because
# embeddings, lm_head and the vision tower stay bf16 — bigger than the Q6/Q8 GGUFs they pair with.

[global]
image              = vllm/vllm-openai-rocm:v0.32.0  # = `latest` on 2026-10-18 (ROCm 10.0, 11.6 GB download); update = bump this + `vllmctl.py pull`
container          = vllm
port               = 11348                           # whisper = 11345/11346, kokoro = 11347
models-dir         = /home/anon/software/models      # mounted at /models (ro) — share the GGUF dir
hf-cache           = /home/anon/.cache/huggingface   # mounted at /root/.cache/huggingface (downloads + token)
router             = http://127.0.0.1:11343          # llama.cpp router root (for the resident-model gate)
reserve-gb         = 2.0                             # abort if projected free VRAM < this (bench.py floor 1.5, target 2.5)
drain-gb           = 4.0                             # "card is clean" threshold before starting (same as bench.py)
health-timeout     = 1800                            # s — first start compiles Triton kernels, be patient
gpu-memory-utilization = 0.75                        # default; presets override
# REQUIRED on this card (✓ verified 2026-10-21): v0.32.0 ships no gfx1102 code objects — without the
# override every GPU kernel launch aborts the process SILENTLY (the engine core just vanishes, the
# API server reports "Failed core proc(s): {}"). gfx1100 (RX 7900) code runs on gfx1102, same RDNA3 ISA.
env.HSA_OVERRIDE_GFX_VERSION = 11.0.0
# Attention backend knobs (ROCm; names moved between versions — `vllmctl.py exec vllm serve --help`):
# env.VLLM_ATTENTION_BACKEND = TRITON_ATTN           # the default on RDNA; set explicitly if a hang points at attention

# ─── 1. smoke test — tiny, certain to be supported; proves docker + GPU + image + override ────
# ✓ 2026-10-21: 154.9 t/s decode · 1462 t/s prefill · TTFT 0.02 s · 7.7 GB used incl. a 2.9 GB desktop.
[Qwen3-0.6B-smoke]
model              = Qwen/Qwen3-0.6B
gpu-memory-utilization = 0.30
max-model-len      = 4096
x-weights-gb       = 1.5

# ─── 2. gemma-4-E4B — ✗ DOES NOT FIT vLLM ON THIS CARD (2026-10-21), tombstone kept for the math ──
# Pairs with llama.cpp [gemma-4-E4B-it-UD-Q8_K_XL] (57 t/s with MTP, 96k ctx). google/gemma-4-E4B-it-qat-w4a16-ct
# is 11.5 GB, and the second try (text-only, batch 2048, @0.73 on a 2.1 GB desktop) ended with
# "Available KV cache memory: -4.44 GiB" after an OOM on a 5,637,144,576-byte allocation. That is exactly
# Gemma 4's per-layer embedding table: 262144 vocab × 42 layers × 256 × 2 bytes (bf16). vLLM holds it
# in bf16 AND needs a second buffer of the same size → ~16 GiB before any KV block. No util, batch or
# desktop fixes that; the GPTQ/AWQ alternatives carry the same bf16 table. llama.cpp fits because the
# GGUF stores the table quantized, once. A different image changes nothing here.
# [gemma-4-E4B-it-w4a16]
# model              = google/gemma-4-E4B-it-qat-w4a16-ct
# gpu-memory-utilization = 0.78
# max-model-len      = 16384
# max-num-seqs       = 4
# max-num-batched-tokens = 2048
# limit-mm-per-prompt = {"image": 0, "audio": 0, "video": 0}
# x-weights-gb       = 11.5

# ─── 3. Qwen3.5-9B — pairs with llama.cpp [Qwen3.5-9B-UD-Q6_K_XL] (32 t/s with MTP, 32k ctx) ──
# Red Hat's llm-compressor int4 (compressed-tensors). 10.95 GB + a 0.49 GB MTP head file.
# Alternative (real AWQ, Triton kernels on ROCm): QuantTrio/Qwen3.5-9B-AWQ — 12.4 GB, too big for 0.78.
# The last viable "same model" pair: no per-layer table, but a 248k vocab → the fp32 logits buffer
# during profiling is max-num-batched-tokens × 248k × 4 B (2048 → 2.0 GB, 512 → 0.5 GB). Start small.
# On a 2.1 GB desktop the projection allows 0.75 (`--set gpu-memory-utilization=0.75`) → ~1.9 GB for
# activations + graphs + KV. If "No available memory for the cache blocks" again: `--set enforce-eager=true`
# (drops the CUDA-graph memory, costs decode speed), then a cleaner desktop for 0.78.
# ✓ RUNS 2026-10-21 @0.80 (0.75 was 0.11 GiB short of KV room): 29.4 t/s decode · 181 prefill · TTFT 0.17 ·
#   decode@8k 16.0 · 4-stream aggregate 60.6 · 12.5 GB used on a 0.6 GB desktop. Same evening, llama.cpp
#   [Qwen3.5-9B-UD-Q6_K_XL] with its MTP draft: 61.3 · 114 · 0.27 · 51.1 · 85.4 · 10.9 GB. See README.
[Qwen3.5-9B-w4a16]
model              = RedHatAI/Qwen3.5-9B-quantized.w4a16
gpu-memory-utilization = 0.80    # needs a ≤ 0.6 GB desktop (projection: 2.9 GB free); 0.75 fails on KV room
max-model-len      = 16384
max-num-seqs       = 4
max-num-batched-tokens = 512     # caps the profiling logits buffer (prefill t/s cost — note it in the ledger)
limit-mm-per-prompt = {"image": 0, "video": 0}   # text-only bench (Qwen3.5 has no audio input)
x-weights-gb       = 11.0
# NEXT (the one knob that could change the verdict): MTP speculative decoding — the repo ships
# model_mtp.safetensors and llama.cpp's 2× comes from exactly this. Costs ~0.5 GB weights + draft
# state → try @0.82 on a clean desktop. Syntax per `vllmctl.py exec vllm serve --help`:
#   ./vllmctl.py up Qwen3.5-9B-w4a16 --set gpu-memory-utilization=0.82 --set 'speculative-config={"method": "mtp", "num_speculative_tokens": 2}'
# speculative-config = {"method": "mtp", "num_speculative_tokens": 2}