# llama.cpp model config — AMD RX 7600 XT (ROCm) · 16 GB (17.16 GB total) # ═════════════════════════════════════════════════════════════════════════════ # ⚠⚠ VRAM SAFETY — READ THIS. This GPU ALSO DRIVES THE DISPLAY. If a model asks for # more VRAM than is free, ROCm does NOT OOM cleanly — it spills into system RAM # (GTT), which starves the desktop and FREEZES THE WHOLE PC (reboot required). # → TARGET ≥ ~2.5 GB free (≤ ~14.5 GB used). HARD FLOOR 1.5 GB (bench.py guard stops there). # Idle desktop uses ~1.3 GB of 17.16 — but a browser/leftover model can hold 1-3 GB more, # so measure with a CLEAN baseline (bench.py prints "baseline used"; close GPU apps first; # sleep-idle keeps a model resident up to 5 min after last use). # → Verify with bench.py → ledger: bench-results.md (latest) + bench-history.md (all runs). # → If a model is tight: LOWER ctx-size, or RAISE n-cpu-moe (MoE), or use a smaller quant. # The values below are deliberately CONSERVATIVE after freeze incidents — raise ctx # only after bench.py confirms the headroom. # ═════════════════════════════════════════════════════════════════════════════ # Speed notes: decode is memory-bandwidth-bound (~288 GB/s). Prefer MoE (few active # params). For MoE that doesn't fit, n-cpu-moe offloads expert FFNs to CPU (48 GB RAM) — # this is FASTER than letting VRAM spill to GTT, and frees VRAM. threads = 6 (5600X cores). # Runtime: `sudo rocm-smi --setperflevel high`; `watch -n1 rocm-smi --showmeminfo vram`. # ───────────────────────────────────────────────────────────────────────────── # Coding models # ───────────────────────────────────────────────────────────────────────────── [Qwen3-Coder-Next-UD-IQ3_XXS] # ⭐ Big-context coder: Qwen3-Coder-Next 80B-A3B (3B active), Qwen3-Next hybrid attn → cheap KV. # Weights ~30 GB: ALL experts on CPU (n-cpu-moe 99), only attention+KV on GPU → low VRAM, safe. # Needs a CURRENT llama.cpp build (Qwen3-Next arch). ⬇ DOWNLOAD unsloth/Qwen3-Coder-Next-GGUF:UD-IQ3_XXS. # Froze the PC before at n-cpu-moe 48 / ctx 131072 (overcommit) — keep it conservative, verify, then raise ctx. model = /home/anon/software/models/Qwen3-Coder-Next-UD-IQ3_XXS.gguf ctx-size = 131072 # ✓ VERIFIED @131072/moe99 (2026-07-16): 15.7 t/s, 6.0 GB used / 11.2 free — # full 128k at a comfortable margin. Settled. n-gpu-layers = 99 n-cpu-moe = 99 # ALL experts on CPU — settled AGAIN 2026-08-08 after a two-round # experiment, CLOSED: GPU experts barely move this model. # moe56 (04:11) = NO-OP: 6.0 used ≈ moe99's 5.8, all numbers flat → # only ~48 MoE layers exist; moe ≥ 48 ≡ all-on-CPU. (Also: the old # "moe48/131k froze the PC" lore was ctx-KV overcommit in a pre- # quantized-KV config, not expert placement.) # moe40 (04:37) = 8 layers on GPU, 10.2 used / 6.9 free: prefill # 33→36 (+9%), decode 16.4 (noise) — 4.2 GB for a rounding error. # The shallow curve says don't chase it: cache-reuse already makes # growing sessions incremental, and the 6 GB footprint IS the niche. # MEASURED (07-16): moe99 = 14.3-16.4 t/s vs moe36@65536 = 16.9. threads = 6 # no-mmap TRIED AND REVERTED 2026-07-18: llama.cpp's log hint suggested it (CPU experts + # mmap = page-fault churn), but reading 26 GB fully resident exceeds the router's load # patience → "failed to load". Retry only if the router ever gets a load-timeout knob. flash-attn = on cache-type-k = q8_0 cache-type-v = q4_0 batch-size = 2048 ubatch-size = 256 # small compute buffer = less VRAM cache-reuse = 256 defrag-thold = 0.1 temp = 0.7 # unsloth default 1.0/top-p0.95/top-k40/min-p0.01 — lower = more deterministic top-p = 0.95 top-k = 40 min-p = 0.01 jinja = on sleep-idle-seconds = 300 # ─── RETIRED 2026-07-16: [DeepSeek-Coder-V2-Lite-Instruct-Q8_0] ────────────────────────── # Mid-2024 model, outclassed by GLM-4.7-Flash + both Qwen coders. Bench kept skipping it # (~15.3 GB fixed footprint; measured 2026-07-03 that ctx/moe knobs barely move it — only fix # was a Q4_K_M requant, not worth it as the 4th-best coder). Removed from opencode/pi/aichat. # File can be deleted from the models dir: DeepSeek-Coder-V2-Lite-Instruct-Q5_K_M.gguf # ─── RETIRED 2026-08-06: [Qwen3-Coder-30B-Instruct-IQ4_XS] ─────────────────────────────── # Lost the 2026-08-06 quant race to UD-Q3_K_XL@moe10 (31.3 vs 23.0 t/s, +36%, three # consistent sweeps; it also failed twice to load on ~3.2 GB evening baselines at moe20 = # 14.1-14.3 used). Clients rewired to Qwen3-Coder-30B-Instruct-UD-Q3_K_XL same day — the # extended quality trial was deliberately skipped (user call; revert path = git, one commit). # ⚠ OWUI chats/presets saved against this id need re-picking after the restart. # File can be deleted from the models dir: Qwen3-Coder-30B-A3B-Instruct-IQ4_XS.gguf # Reference numbers: moe24 = 21.1 t/s, 11.9/5.3 · moe20 = 23.2 (2026-07-16), 22.0-23.0 (2026-08-05/06) [Qwen3-Coder-30B-Instruct-UD-Q3_K_XL] # ⭐ MAIN AGENT CODER — took the slot 2026-08-06 after winning the quant race: 31.3 t/s @moe10 # vs IQ4_XS@moe20 = 23.0 (+36%, three consistent sweeps); Q3_K_S dominated (tombstone below). # Same 32k ctx and sampling as the retired IQ4_XS — clients only needed the id swap. # ⚠ QUALITY TRIAL SKIPPED (user call, 2026-08-06): Q3-on-MoE damage looks like wrong API # signatures / off-by-one logic, NOT word salad — if opencode sessions smell off, the # IQ4_XS preset + client wiring are one `git revert` away (and the .gguf may still be on # disk). UD protects attn/shared tensors; the experts are still 3-bit. # RACE NUMBERS: moe24 = 22.2 · moe16 = 25.1 (07-16) / 25.5 / 26.3 · moe10 = 31.3 (15.0 used / # 2.1 free @3.1 GB baseline; replicate 30.9 @1.8 free under a 13.9 leftover-model baseline). model = /home/anon/software/models/Qwen3-Coder-30B-A3B-Instruct-UD-Q3_K_XL.gguf ctx-size = 32768 n-gpu-layers = 99 n-cpu-moe = 12 # ✓ VERIFIED @moe12 (2026-08-07, 2.9 GB EVENING baseline): 27.0 t/s, # 14.2 used / 3.0 free — loaded fine in exactly the regime that killed # IQ4_XS@moe20 twice; the insurance works. Cost vs moe10 (31.3 clean race) # was ~4 t/s, more than the ~1 estimated — evening compute contention # likely eats 2-3 of that; confirmed: CLEAN (2026-08-08 02:45, 1.9 # baseline) = 29.5 t/s, 13.2 used / 4.0 free — evening cost was indeed # contention. Settled: moe12 = 29.5 clean / 27.0 evening, always loads. threads = 6 flash-attn = on cache-type-k = q8_0 cache-type-v = q4_0 batch-size = 2048 ubatch-size = 1024 # ✓ SETTLED 2026-08-08: prefill 74 → 89-92 t/s (+24%), decode unchanged, # buffer +0.5 GB. FINAL GATE PASSED (14:43 sweep, 3.0 GB dirty baseline = # evening-class): loaded and ran at 14.6 used / 2.6 free. If a worse # evening (3.3+) ever fails the load: revert to 512, or pay one moe # step (12→13). cache-reuse = 256 defrag-thold = 0.1 temp = 0.15 top-p = 0.8 top-k = 20 min-p = 0 repeat-penalty = 1.05 jinja = on sleep-idle-seconds = 300 # ─── RETIRED 2026-08-06: [Qwen3-Coder-30B-Instruct-Q3_K_S] ─────────────────────────────── # Dominated in the quant race: 30.9 t/s @moe8 (15.2 used / 2.0 free, replicate 30.9) = a tie # with UD-Q3_K_XL@moe10 (31.3) at equal headroom, from a strictly worse quant (plain Q3_K_S, # no UD-protected attn/shared tensors). No axis where it wins. Was never client-wired. # File can be deleted from the models dir: Qwen3-Coder-30B-A3B-Instruct-Q3_K_S.gguf # ───────────────────────────────────────────────────────────────────────────── # General Qwen models (35B-A3B is fast MoE; the dense 27Bs are inherently VRAM-tight) # ───────────────────────────────────────────────────────────────────────────── [Qwen3.6-35B-A3B-MTP-UD-IQ3_XXS] # ⭐ Fast NON-THINKING daily driver: Qwen3.6-35B-A3B (MoE, 3B active) — the "bigger Qwen3 # Instruct". Compact precise answers, no reasoning preamble. (Alias kept for clients.) # File is flat + named without "MTP" (unsloth MTP-repo naming). If the load log complains # there is no MTP head in this file, comment the two spec- lines out (plain repo file). model = /home/anon/software/models/Qwen3.6-35B-A3B-UD-IQ3_XXS.gguf mmproj = /home/anon/software/models/mmproj-Qwen3.6-35B-A3B-F16.gguf # vision ✓ VERIFIED 2026-07-18: # 13.9 used / 3.3 free (2.8 dirty baseline), decode unchanged at 36-37 t/s. spec-type = draft-mtp spec-draft-n-max = 2 reasoning-budget = 0 # hard-disable thinking → instant compact answers. Delete to allow /think. # ⚠ if startup says "option 'reasoning_budget' not recognized" → remove this line. ctx-size = 24576 # raised from 16384 (bench @16384: 4.8 GB free) — bench to confirm ≥2.5 free n-gpu-layers = 99 n-cpu-moe = 16 # MEASURED @24: 19.8 t/s (MTP working!), 10.6 used / 6.6 free → pull experts onto GPU threads = 6 flash-attn = on cache-type-k = q8_0 cache-type-v = q4_0 batch-size = 2048 ubatch-size = 512 cache-reuse = 256 defrag-thold = 0.1 temp = 0.7 top-p = 0.8 top-k = 20 min-p = 0 repeat-penalty = 1.05 jinja = on sleep-idle-seconds = 120 [Qwen3.6-35B-A3B-Thinking] # Same GGUF as above with reasoning ENABLED (no reasoning-budget line) — fills the "thinking # model" slot at zero disk cost. Qwen thinking-mode sampling: temp 0.6 / top-p 0.95, no # repeat-penalty. Slower per answer (reasoning preamble) — use for hard problems. model = /home/anon/software/models/Qwen3.6-35B-A3B-UD-IQ3_XXS.gguf mmproj = /home/anon/software/models/mmproj-Qwen3.6-35B-A3B-F16.gguf # vision ✓ VERIFIED 2026-07-18: # 13.9 used / 3.3 free (2.8 dirty baseline), decode unchanged at 36-37 t/s. spec-type = draft-mtp spec-draft-n-max = 2 ctx-size = 24576 # thinking eats ctx — same footprint as the non-thinking preset n-gpu-layers = 99 n-cpu-moe = 16 threads = 6 flash-attn = on cache-type-k = q8_0 cache-type-v = q4_0 batch-size = 2048 ubatch-size = 512 cache-reuse = 256 defrag-thold = 0.1 temp = 0.6 top-p = 0.95 top-k = 20 min-p = 0 jinja = on sleep-idle-seconds = 120 # ─── RETIRED 2026-07-03: [Qwen3.6-27B-UD-IQ3_XXS] ──────────────────────────────────────── # The 27B files (plain AND MTP) were deleted from the models dir — preset removed so the # router doesn't advertise a dead model. To bring it back: re-download (MTP-repo files are # named WITHOUT "MTP"), restore the preset, re-add the id to opencode/pi/aichat lists. # ─── RETIRED 2026-07-16: [Qwen3.5-27B-UD-Q3_K_XL] ──────────────────────────────────────── # Dense 27B: 16.2 GB used / 0.9 free even @8192 (under the 1.5 floor) and ~10 t/s ceiling on # this card. Role covered by Qwen3.6-35B-A3B (MoE, 2× faster). Removed from opencode/pi/aichat. # File can be deleted from the models dir: Qwen3.5-27B-UD-Q3_K_XL.gguf [Qwen3.5-9B-UD-Q6_K_XL] # Small fast Qwen + MTP. Replaces Qwen3.5-9B-Q8_0 (retired 2026-07-16: Q8 was quality-overkill # and bandwidth-slow at 17.2 t/s — delete Qwen3.5-9B-Q8_0.gguf; also delete the interim # Qwen3.5-4B-UD-Q6_K_XL.gguf, superseded by this). # ⬇ FILE MUST COME FROM unsloth/Qwen3.5-9B-MTP-GGUF (8.99 GB — MTP head embedded, filename # identical to the plain repo!). If the load log complains there is no MTP head in this # file, it's the plain-repo file — comment the two spec- lines out. # ✓ VERIFIED (2026-07-16): 32.2 t/s (MTP working — nearly 2× the Q8's 17.2), prefill 83, # 12.4 used / 4.8 free @32768. TTFT ~3.7 s = the model thinking (reasoning streams first). model = /home/anon/software/models/Qwen3.5-9B-UD-Q6_K_XL.gguf mmproj = /home/anon/software/models/Qwen3.5-9B-MTP-mmproj-F16.gguf # vision (from the MTP repo) spec-type = draft-mtp spec-draft-n-max = 3 # ⚗→✓ KEPT (04:11): 33.8 t/s vs 32.0-33.1 @n=2 — small real win. # Revert to 2 if a future sweep drops below 32. ctx-size = 32768 # raise toward 65536 once bench confirms free ≥2.5 GB n-gpu-layers = 99 flash-attn = on cache-type-k = q8_0 cache-type-v = q4_0 # q4 V to save VRAM batch-size = 2048 ubatch-size = 512 cache-reuse = 256 defrag-thold = 0.1 temp = 0.7 top-p = 0.8 top-k = 20 min-p = 0 repeat-penalty = 1.05 jinja = on sleep-idle-seconds = 60 # ─── RETIRED 2026-07-16: [Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-IQ3_M] ────────────── # Dense: 10 t/s, 2.0 GB free (under the 2.5 target) and unfixable — bandwidth-bound, smaller # quant would cost the quality that justified it. Removed from opencode/pi/aichat. # File can be deleted from the models dir: Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-IQ3_M.gguf # ───────────────────────────────────────────────────────────────────────────── # Gemma models # ───────────────────────────────────────────────────────────────────────────── [gemma-4-26B-A4B-it-UD-IQ4_XS] # MoE (4B active) + MTP + vision. ~13.5 GB weights → tight. Projector on CPU (no-mmproj-offload). # From unsloth/gemma-4-26B-A4B-it-qat-GGUF: mmproj-F16.gguf + mtp-gemma-4-26B-A4B-it.gguf. # ⬇ SOURCES — local name ← upstream (PLAIN repo unsloth/gemma-4-26B-A4B-it-GGUF; the qat # repo has NO IQ4_XS, its only main is qat-UD-Q4_K_XL 14.25 GB — see decision note below): # gemma-4-26B-A4B-it-UD-IQ4_XS.gguf ← gemma-4-26B-A4B-it-UD-IQ4_XS.gguf (13.6 GB) # ✓ current: the 2026-08-05 dl has the Jul 17 template # mmproj-gemma-4-26B-A4B-it-F16.gguf ← mmproj-F16.gguf (1.19 GB) # mtp-gemma-4-26B-A4B-it.gguf ← MTP/mtp-gemma-4-26B-A4B-it-Q8_0.gguf (462 MB) # Keep LOCAL names stable; re-fetch the upstream names above when repos update. Pair main # and drafter from the SAME repo — the two repos' drafters are different extractions. # ✓ FIXED 2026-08-08 03:15: matched plain-repo sidecars cured the "Invalid input batch." # 500s — root cause CONFIRMED as the cross-repo drafter mismatch (old qat-repo mtp under # the new plain main). Re-benched 32.8 t/s @moe13, 12.1 used / 5.0 free, MTP ✓ — same # speed as pre-update (32.7), template fix now in place. The *.gguf.bak sidecars are # deletable once vision + opencode tool-calling are smoke-tested. # (QAT quality upgrade = separate decision: main → qat-UD-Q4_K_XL, +0.7 GB, moe 13→~16, and # a section/client id rename round. Not part of this fix.) # ⚠ vision + MTP together is heavy — if it OOMs/freezes, add n-cpu-moe or drop MTP (keep vision). model = /home/anon/software/models/gemma-4-26B-A4B-it-UD-IQ4_XS.gguf mmproj = /home/anon/software/models/mmproj-gemma-4-26B-A4B-it-F16.gguf model-draft = /home/anon/software/models/mtp-gemma-4-26B-A4B-it.gguf spec-type = draft-mtp spec-draft-n-max = 2 reasoning-budget = 0 # added 2026-08-09: the Jul-17 official template makes gemma-4 DELIBERATE # on hard prompts (probe: 700 tok = 100% reasoning_content, content # EMPTY, finish=length) — that ate pi's 4096 maxTokens cap and returned # truncated nothing. Same fix as the 35B daily driver. ⚠ if startup says # option not recognized, remove; if it loads but a hard probe still emits # reasoning_content, the gemma template ignores the budget → remove and # raise client maxTokens to 8192 instead. ctx-size = 24576 # MEASURED @16384/moe16: 29.6 t/s, 10.7 used / 6.5 free → sliding-window KV is # cheap, take the ctx back (clients already assume 24576) # n_ctx_train = 262144 (08-08 load log) — huge ctx headroom exists if a # long-context vision need ever appears; bench stepwise, free ≥ 2.5. n-gpu-layers = 99 n-cpu-moe = 13 # MEASURED @16: 26.4 t/s, 11.9 used / 5.2 free → pull ~3 expert layers onto GPU for speed threads = 6 flash-attn = on jinja = on # OpenAI-style tool calling (opencode build/plan default to this preset) cache-type-k = q8_0 cache-type-v = q4_0 batch-size = 2048 ubatch-size = 512 cache-reuse = 256 defrag-thold = 0.1 no-mmproj-offload = true # vision encoder on CPU → frees ~1 GB temp = 1.0 top-p = 0.95 top-k = 64 sleep-idle-seconds = 60 [gemma-4-E4B-it-UD-Q8_K_XL] # ~5-7 GB — small/fast, lots of headroom, safe at big ctx. # Vision: the HF repo ships mmproj-{BF16,F16,F32}.gguf — use F16 (990 MB), saved AS the exact # path below (rename it; if the path doesn't exist the model fails to load / vision silently # missing depending on router). Verify: startup log shows the mmproj load, and an image request # answers instead of erroring "image input is not supported". # NOTE: repo got Google's chat-template update ~2026-05 — if the main gguf predates that, re-download it too. model = /home/anon/software/models/gemma-4-E4B-it-UD-Q8_K_XL.gguf mmproj = /home/anon/software/models/mmproj-gemma-4-E4B-it.gguf model-draft = /home/anon/software/models/mtp-gemma-4-E4B-it.gguf # ✓ MEASURED (2026-07-18): 57.3 t/s — # +50% over 38.4 without the draft, 10.5 used / 6.6 free. Fastest model on the box. spec-type = draft-mtp spec-draft-n-max = 2 reasoning-budget = 0 # added 2026-08-09, same reason as the 26B: gemma-4 deliberates on hard # prompts since the Jul-17 template (this is what caused the 20-second # aichat -e "think" preambles on 2026-08-05). Fast models answer NOW. # ⚠ same caveats as the 26B — remove if not recognized / not honored. ctx-size = 98304 # ⚗→✓ CLOSED 2026-08-08: 58.2-60.4 t/s, 10.3-10.6 used / 6.5-6.9 free # @98304, and n_ctx_train = 131072 confirmed (upstream config.json # max_position_embeddings) — 98304 is inside training range, no RoPE # risk. 131072 is the hard ceiling if ever needed (~+0.5-1 GB KV). n-gpu-layers = 99 flash-attn = on jinja = on # OpenAI-style tool calling (doc-writer agent defaults to this preset) cache-type-k = q8_0 cache-type-v = q8_0 batch-size = 2048 ubatch-size = 1024 cache-reuse = 256 defrag-thold = 0.1 temp = 1.0 top-p = 0.95 top-k = 64 sleep-idle-seconds = 30 # ───────────────────────────────────────────────────────────────────────────── # GLM # ───────────────────────────────────────────────────────────────────────────── [GLM-4.7-Flash-UD-Q4_K_XL] # 30B-A3B MoE, primary coder. Upgraded UD-Q3_K_XL (~14 GB) → UD-Q4_K_XL (17.5 GB, unsloth's # recommended quant) 2026-07-16, benched clean same day → the old GLM-4.7-Flash-UD-Q3_K_XL.gguf # can be deleted. # Tuning trajectory: Q3@moe9 = 28.2 t/s → Q4@moe24 = 17.9 (measured 2026-07-16) → now moe18, # lower stepwise toward 16 while bench keeps free ≥2.5 GB. model = /home/anon/software/models/GLM-4.7-Flash-UD-Q4_K_XL.gguf ctx-size = 24576 # MEASURED @16384/moe24 (Q3): 9.5 GB used, 7.6 free # n_ctx_train = 202752 (08-08 load log) — MLA KV is compact, ctx could # go far higher if opencode subagent sessions ever outgrow 24k. n-gpu-layers = 99 n-cpu-moe = 22 # lowered 24→22 on 2026-08-08: both step-down gates passed post-KV-fix — # clean sweep @24 = 20.5 t/s, 12.0 used / 5.2 free (02:45, 1.9 baseline); # elevated-baseline load @24 = 18.7, 13.0/4.2 (@2.9). moe22 est ~21.5 # clean / ~19.5-20 evening, ~12.7 used clean / ~14.0 evening → inside the # coder-proven evening envelope (14.2 loads). ✓ CLEAN VERIFIED (02:58, # 2.0 baseline): 21.5 t/s, 12.8 used / 4.4 free — est was 21.5, exact. # Evening observation pending; revert to 24 if an evening bench shows # free < 2.5. moe20 stays gated (~14.4-14.7 evening = borderline vs the # coder's 15.2 evening death). # (08-05/07 "failures" were the mixed-KV rejection below, not VRAM.) # Trajectory: Q3@moe9 = 28.2 → Q4@moe24 = 17.9 (07-16) → 18.7/20.5 post-KV-fix. threads = 6 flash-attn = on cache-type-k = q8_0 cache-type-v = q8_0 # was q4_0 — BROKE 2026-08-08: a llama.cpp upgrade (the one deprecating # defrag-thold) enforces same-type K/V for this model (deepseek2/MLA # conversion; MLA KV is compact, mixed types unsupported). Every load # since failed at context creation regardless of VRAM. Only THIS preset # was affected — the other k-q8/v-q4 presets are regular-attention archs. # If q8/q8 still refuses: delete both cache-type lines (f16 default; MLA # KV is small enough that the cost is minor at 24576 ctx). batch-size = 2048 ubatch-size = 512 cache-reuse = 256 defrag-thold = 0.1 # MTP: GLM-4.7-Flash HAS an MTP head upstream (num_nextn_predict_layers: 1) but the GGUF # conversion (deepseek2 arch) drops those tensors and llama.cpp draft-mtp only supports # Qwen3.5/3.6 + Gemma 4 so far (PRs #22673/#23398). No mtp-*.gguf exists to download — # recheck when llama.cpp lands GLM MTP support. temp = 0.6 # unsloth rec: 0.7/top-p 1.0 (tool-calling) or 1.0/0.95 (general); no repeat-penalty top-p = 0.95 min-p = 0.01 jinja = on reasoning-preserve = true # ⚗ option ACCEPTED by the build (loaded + ran 20.9 t/s, 04:11 sweep). # BEHAVIOR flag — judge multi-turn subagent quality in opencode over the # next days; remove if reasoning bleed ever pollutes final answers. sleep-idle-seconds = 60 # ───────────────────────────────────────────────────────────────────────────── # Utility & candidates # ───────────────────────────────────────────────────────────────────────────── [Qwen3-Embedding-0.6B] # Embeddings-only, for RAG/semantic search (OWUI etc. via /v1/embeddings) — NOT a chat model, # so it is deliberately absent from the opencode/pi/aichat chat lists. # Qwen3-Embedding requires last-token pooling. ~0.7 GB — tiny, loads fast. # (Q8_0 is effectively lossless for a 0.6B embedder — the f16 file is redundant, delete it.) # CPU-ONLY on purpose: a 0.6B embeds fast on 6 cores, and 0 VRAM means it can never stack # with a chat model into an overcommit (and never suffers/causes eviction pressure). model = /home/anon/software/models/Qwen3-Embedding-0.6B-Q8_0.gguf embeddings = true pooling = last ctx-size = 8192 n-gpu-layers = 0 flash-attn = on batch-size = 8192 # embed long chunks in one pass ubatch-size = 1024 threads = 6 sleep-idle-seconds = 60 [gpt-oss-20b] # OpenAI gpt-oss-20b — MoE 21B / 3.6B active, reasoning + tool use (enabled 2026-07-16, NOT # BENCHED yet). OpenAI post-trained it in MXFP4, so there is exactly ONE sensible GGUF: # ggml-org/gpt-oss-20b-GGUF → gpt-oss-20b-mxfp4.gguf (~12.1 GB). Requants are never better — # the old gpt-oss-20b-Q3_K_M.gguf can be deleted. # Harmony chat template → jinja required. Default KV types kept (gpt-oss attention sinks have # had issues with quantized KV — verify before adding cache-type lines). model = /home/anon/software/models/gpt-oss-20b-mxfp4.gguf ctx-size = 65536 # ⚗→✓ VERIFIED @65536 (04:11): 37.3/37.4 t/s unchanged, 14.0 used / 3.2 # free — the fast long-context option is real (37 t/s vs Coder-Next 16). # 131072 is OFF the table: 32k→65k cost +1.2 GB (KV slope steepens past # the sliding window) → 131k would land under 2.5 free. Client caps # raised 32k→64k on 2026-08-08. n-gpu-layers = 99 n-cpu-moe = 4 # ✓ SETTLED. MEASURED @6: 31.3 t/s · @4 (2026-07-16): 38.1 t/s, 12.7 used / # 4.5 free — fastest model in the lineup. TTFT (~3-7 s) is the model THINKING # (harmony reasoning streams as reasoning_content first) — not a slow load. threads = 6 flash-attn = on batch-size = 2048 ubatch-size = 512 cache-reuse = 256 defrag-thold = 0.1 temp = 1.0 top-p = 1.0 jinja = on sleep-idle-seconds = 120 [gpt-oss-20b-low] # Same GGUF, reasoning effort LOW — zero disk cost (like the 35B-Thinking alias, inverted). # Cuts the multi-second thinking preamble for light/interactive use; keep [gpt-oss-20b] # (default = medium effort) for hard problems. ⚠ if startup says the option is not # recognized, the llama.cpp build predates chat-template-kwargs — comment that line out. model = /home/anon/software/models/gpt-oss-20b-mxfp4.gguf chat-template-kwargs = {"reasoning_effort":"low"} ctx-size = 65536 # ⚗→✓ VERIFIED @65536 (04:11): 37.3/37.4 t/s unchanged, 14.0 used / 3.2 # free — the fast long-context option is real (37 t/s vs Coder-Next 16). # 131072 is OFF the table: 32k→65k cost +1.2 GB (KV slope steepens past # the sliding window) → 131k would land under 2.5 free. Client caps # raised 32k→64k on 2026-08-08. n-gpu-layers = 99 n-cpu-moe = 4 threads = 6 flash-attn = on batch-size = 2048 ubatch-size = 512 cache-reuse = 256 defrag-thold = 0.1 temp = 1.0 top-p = 1.0 jinja = on sleep-idle-seconds = 60 # ───────────────────────────────────────────────────────────────────────────── # Notes # ───────────────────────────────────────────────────────────────────────────── # n-cpu-moe N : offload the first N MoE layers' experts to CPU. Higher = less VRAM (safer) but # slower. On this display-driving GPU, a bit of offload is FAR better than a freeze. # Only applies to MoE models (dense models ignore it — reduce ctx instead). # override-tensor : finer control, e.g. `blk\.(1[2-9]|[2-9][0-9])\.ffn_.*_exps=CPU`. # After ANY change: reload the server and run bench.py — keep every "VRAM free" ≥ ~2.5 GB.