[Sync] replace the flat layout with the unified stow tree
Supersedes the old flat .config/ layout (last published 2026-06-28) with the private repo's structure: one shared base plus per-host overlays. - packages: common/ gui/ lw/ fl/ wm/ plus install.sh and bin/ tooling (dotsync, reconcile-hyde.sh) - new README covering the layout, deploy order and the HyDE dependency - current HyDE waybar rig (layouts/, cava), pi agent extensions, claude/ config, tmux, presenterm, aichat roles - fish: kp (keepassxc-cli + fzf picker, db path from $KP_DB) and bind_M_n_history (alt+1..9 recalls the nth history entry) - drops cruft that should never have been tracked: the duplicate top-level .pi/ copy, btop.log, zellij config.kdl.bak, fish_variables - .pi/agent/auth.json is gitignored; auth.json.example ships instead Host-specific work sessions and the personal backlog stay in the private tree. Endpoint locators in the llamacpp/whisper guides are placeholders ($SERVER, <own-domain>) — the guides themselves stay, since they are the useful part.
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
# LW HOST OVERLAY — full-file copy of common/.config/aichat/config.yaml plus the `lw` client
|
||||
# (this laptop's own llama.cpp server at 127.0.0.1:11343; see lw/.config/llamacpp/README.md).
|
||||
# Stowed with --override so it shadows the common file on lw only. When the common config
|
||||
# changes, mirror the change here.
|
||||
# see https://github.com/sigoden/aichat/blob/main/config.example.yaml
|
||||
keybindings: vi
|
||||
editor: nvim
|
||||
model: local:Qwen3-Coder-30B-Instruct-UD-Q3_K_XL
|
||||
|
||||
# Sessions: persist REPL sessions and keep more history before summarizing.
|
||||
# The default compress_threshold (4000) summarizes far too early for 24k+ windows.
|
||||
save_session: true
|
||||
compress_threshold: 16000
|
||||
|
||||
# REPL prompts show live context usage (needs max_input_tokens, set per model below)
|
||||
left_prompt: '{color.green}{?session {session}{?role /}}{role}{color.cyan}{?rag @{rag}}{color.reset}> '
|
||||
right_prompt: '{color.purple}{?session {consume_tokens}/{max_input_tokens} }{color.reset}'
|
||||
|
||||
# NOTE: temperature/top_p are intentionally unset — the LAN server applies tuned
|
||||
# per-model sampling via --jinja (e.g. GLM 0.6/0.95); a global value would clobber it.
|
||||
|
||||
# max_input_tokens = real ctx-size (from the router) minus output headroom.
|
||||
clients:
|
||||
# This laptop's own CPU server (llamaserver abbr; works everywhere, no network needed).
|
||||
# Speeds are benched decode t/s — see lw/.config/llamacpp/README.md roster.
|
||||
- type: openai-compatible
|
||||
name: lw
|
||||
api_base: http://127.0.0.1:11343/v1
|
||||
models:
|
||||
- name: gemma-4-E2B-it-UD-Q4_K_XL
|
||||
max_input_tokens: 3000 # ctx 4096 · ~6 t/s — fastest, quick Q&A
|
||||
- name: Granite-4.0-H-Tiny
|
||||
max_input_tokens: 6000 # ctx 8192 · ~4.8 t/s — speed AND brains
|
||||
- name: Qwen3-1.7B
|
||||
max_input_tokens: 6000 # ctx 8192 · ~4.8 t/s — thinking, snappy
|
||||
- name: Qwen3-4B-Instruct-2507
|
||||
max_input_tokens: 3000 # ctx 4096 · ~3.9 t/s — daily driver
|
||||
- name: Qwen2.5-Coder-3B-Instruct
|
||||
max_input_tokens: 6000 # ctx 8192 · ~3.3 t/s — small coder
|
||||
- name: Qwen3-30B-A3B-Instruct-2507-UD-IQ3_XXS
|
||||
max_input_tokens: 3000 # ctx 4096 · ~2.7 t/s — quality when you can wait
|
||||
- name: Jan-v3-4b
|
||||
max_input_tokens: 3000 # ctx 4096 · ~2.6 t/s — agentic tune
|
||||
- name: gemma-4-E4B-it-UD-Q4_K_XL
|
||||
max_input_tokens: 3000 # ctx 4096 · ~2.5 t/s — quality (vision once mmproj added)
|
||||
# LAN access to the fl box (fast; only reachable on the home network)
|
||||
- type: openai-compatible
|
||||
name: local
|
||||
api_base: http://192.168.0.204:11343/v1
|
||||
models: &lan_models
|
||||
- name: Qwen3-Coder-30B-Instruct-UD-Q3_K_XL
|
||||
max_input_tokens: 30000 # ctx 32768
|
||||
- name: Qwen3-Coder-Next-UD-IQ3_XXS
|
||||
max_input_tokens: 128000 # ctx 131072
|
||||
- name: Qwen3.6-35B-A3B-MTP-UD-IQ3_XXS
|
||||
max_input_tokens: 22000 # ctx 24576
|
||||
supports_vision: true
|
||||
- name: Qwen3.6-35B-A3B-Thinking
|
||||
max_input_tokens: 22000 # ctx 24576
|
||||
supports_vision: true
|
||||
- name: Qwen3.5-9B-UD-Q6_K_XL
|
||||
max_input_tokens: 30000 # ctx 32768
|
||||
supports_vision: true
|
||||
- name: gemma-4-26B-A4B-it-UD-IQ4_XS
|
||||
max_input_tokens: 22000 # ctx 24576
|
||||
supports_vision: true
|
||||
- name: gemma-4-E4B-it-UD-Q8_K_XL
|
||||
max_input_tokens: 62000 # ctx 65536
|
||||
supports_vision: true
|
||||
- name: GLM-4.7-Flash-UD-Q4_K_XL
|
||||
max_input_tokens: 22000 # ctx 24576
|
||||
- name: gpt-oss-20b
|
||||
max_input_tokens: 62000 # ctx 65536
|
||||
- name: gpt-oss-20b-low
|
||||
max_input_tokens: 62000 # ctx 65536
|
||||
# Remote access via duskadiy.com (reachable from anywhere; requires an API key —
|
||||
# $DUSKADIY_API_KEY, from the gitignored conf.d/secrets.fish). The LAN endpoint above is keyless.
|
||||
# Same server, same model ids — reuse the list above via a YAML anchor.
|
||||
- type: openai-compatible
|
||||
name: duskadiy
|
||||
api_base: https://llm.duskadiy.com/api/v1
|
||||
models: *lan_models
|
||||
@@ -0,0 +1,4 @@
|
||||
# --host 127.0.0.1 deliberately: laptop roams onto untrusted wifi and the API has no key.
|
||||
# --cors-origins localhost: llama.cpp PR 25655 recommendation for same-machine use — without it
|
||||
# the default CORS reflects any Origin, letting a malicious page in the local browser call the API.
|
||||
abbr llamaserver "nice -n 19 llama-server --host 127.0.0.1 --port 11343 --cors-origins localhost --models-max 1 --models-preset ~/.config/llamacpp/config.ini"
|
||||
@@ -0,0 +1,4 @@
|
||||
# lw host extras (sourced last by ~/.config/hypr/hyprland.conf)
|
||||
# No discrete GPU on this laptop — no nvidia.conf.
|
||||
|
||||
exec-once = xremap ~/.config/xremap/config.yml # keyboard remaps (lw is the only xremap host)
|
||||
@@ -0,0 +1,21 @@
|
||||
# lw hyprlock sizes — 1920x1080, scale 1.
|
||||
#
|
||||
# gui/.config/hypr/hyprlock.conf ships sizes tuned for wm's 3072x1728 logical panel; these
|
||||
# are those values × 0.625, which keeps the same proportions the layout was designed around
|
||||
# (clock ~13% of screen height, date ~2.5%, meta ~1.5%, logo ~16%).
|
||||
#
|
||||
# Caveat — lw is multi-monitor and hyprlock draws on every output. monitors.conf makes
|
||||
# $mainScreen (eDP-1) 1366x768, so on the built-in panel these read ~40% larger than
|
||||
# intended (clock ≈ 18% of its height). Sized for the 1920x1080 externals on purpose; if
|
||||
# the laptop panel is the one you actually lock on, use the 768-tuned set instead:
|
||||
# $fsClock = 98 $fsDate = 20 $fsMeta = 12 $fsSmall = 11 $fsIcon = 18 $logoSize = 124
|
||||
$fsClock = 138
|
||||
$fsDate = 28
|
||||
$fsMeta = 16
|
||||
$fsSmall = 15
|
||||
$fsIcon = 25
|
||||
$logoSize = 175
|
||||
|
||||
# No $y* overrides: the shared offsets in hyprlock.conf already spread the content to the
|
||||
# edges, and they land the same on both hosts because each host's sizes are the same
|
||||
# proportion of its screen. Override them here if lw ever needs to differ.
|
||||
@@ -0,0 +1,64 @@
|
||||
|
||||
# █▀▄▀█ █▀█ █▄░█ █ ▀█▀ █▀█ █▀█ █▀
|
||||
# █░▀░█ █▄█ █░▀█ █ ░█░ █▄█ █▀▄ ▄█
|
||||
|
||||
# Set your monitor configuration here
|
||||
# See https://wiki.hyprland.org/Configuring/Monitors/
|
||||
# For a sample file, please refer to https://github.com/prasanthrangan/hyprdots/blob/main/Configs/.config/hypr/monitors.t2
|
||||
|
||||
$mainScreen = eDP-1
|
||||
$leftScreen = DP-2
|
||||
$rightScreen = HDMI-A-1
|
||||
|
||||
monitor = $mainScreen, 1366x768@60.14, 0x0, 1 # main screen
|
||||
monitor = $leftScreen, 1920x1080@60, -1920x0, 1 # left
|
||||
monitor = $rightScreen, 1920x1080@60,1366x0, 1 # right
|
||||
|
||||
# monitor=HDMI-A-1,1920x1080@60,-1366x0,1 # hdmi on left
|
||||
# monitor=HDMI-A-1,1920x1080@60,0x-768,1 # hdmi on top
|
||||
# monitor=DP-2,640x480@59.94Hz,0x0,1 # vga
|
||||
|
||||
# monitor = HDMI-A-1, 1366x768@60.00Hz, auto , 1, mirror, eDP-1 #mirror lapscreen
|
||||
# monitor = DP-2, 1920x1080@60.00Hz, auto , 1, mirror, eDP-1 #mirror lapscreen jag3
|
||||
# monitor = eDP-1, 1920x1080@60.00Hz, auto , 1, mirror, HDMI-A-1 #main hdmi, lapscreen mirrors
|
||||
# monitor = eDP-1, 1920x1080@60.00Hz, auto , 1, mirror, DP-2 #main VGA, lapscreen mirrors
|
||||
|
||||
# monitor = , preferred, auto, 1, mirror, DP-2 # dmz projector
|
||||
# monitor = eDP-1, 1920x1080@60, auto , 1, mirror, DP-2 #mirror vga test
|
||||
|
||||
#Main Monitor
|
||||
workspace = 1, monitor:$mainScreen, default:true
|
||||
workspace = 2, monitor:$mainScreen
|
||||
workspace = 3, monitor:$mainScreen
|
||||
workspace = 4, monitor:$mainScreen
|
||||
workspace = 5, monitor:$mainScreen
|
||||
workspace = 6, monitor:$mainScreen
|
||||
workspace = 7, monitor:$mainScreen
|
||||
workspace = 8, monitor:$mainScreen
|
||||
workspace = 9, monitor:$mainScreen
|
||||
workspace = 10,monitor:$mainScreen
|
||||
|
||||
# Second Monitor
|
||||
workspace = 11, monitor:$rightScreen, default:true
|
||||
workspace = 12, monitor:$rightScreen
|
||||
workspace = 13, monitor:$rightScreen
|
||||
workspace = 14, monitor:$rightScreen
|
||||
workspace = 15, monitor:$rightScreen
|
||||
workspace = 16, monitor:$rightScreen
|
||||
workspace = 17, monitor:$rightScreen
|
||||
workspace = 18, monitor:$rightScreen
|
||||
workspace = 19, monitor:$rightScreen
|
||||
workspace = 20, monitor:$rightScreen
|
||||
|
||||
# Third Monitor
|
||||
workspace = 21, monitor:$leftScreen, default:true
|
||||
workspace = 22, monitor:$leftScreen
|
||||
workspace = 23, monitor:$leftScreen
|
||||
workspace = 24, monitor:$leftScreen
|
||||
workspace = 25, monitor:$leftScreen
|
||||
workspace = 26, monitor:$leftScreen
|
||||
workspace = 27, monitor:$leftScreen
|
||||
workspace = 28, monitor:$leftScreen
|
||||
workspace = 29, monitor:$leftScreen
|
||||
workspace = 30, monitor:$leftScreen
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# lw (laptop, smaller screen)
|
||||
font_size 9.0
|
||||
@@ -0,0 +1,13 @@
|
||||
# run with `kitty --session ~/.config/kitty/music.conf`
|
||||
# lw override: smaller split resizes than the shared version, to fit the laptop screen.
|
||||
|
||||
new_tab music
|
||||
layout splits
|
||||
cd ~
|
||||
|
||||
launch --var window=first cava
|
||||
launch --location=hsplit pyradio
|
||||
resize_window taller 10
|
||||
focus
|
||||
launch --location=vsplit pipes-rs
|
||||
resize_window wider 20
|
||||
@@ -0,0 +1,133 @@
|
||||
# Local LLM server — laptop (lw), setup & usage
|
||||
|
||||
llama.cpp in **router mode** on the laptop. This is the **CPU** counterpart of the fl box
|
||||
(`fl/.config/llamacpp/`) — lw has **no discrete GPU**, so everything runs on the CPU and
|
||||
the only resource ceiling is **system RAM** (not VRAM). None of fl's VRAM/freeze rules apply.
|
||||
|
||||
- **Box**: Intel Core i7-5600U (Broadwell 2015, **2 cores / 4 threads**, AVX2) · iGPU HD Graphics
|
||||
5500 (no compute) · **15.49 GiB RAM** · DDR3L-1600 dual-channel: ~20 GB/s theoretical, **~8–10
|
||||
GB/s effective measured** — the dense-decode bottleneck → prefer **small quants + MoE**.
|
||||
- **Endpoint**: `http://127.0.0.1:11343/v1` — **local only** (server binds `127.0.0.1`; deliberate:
|
||||
the laptop roams onto untrusted wifi and the API has no key, unlike the stationary fl box).
|
||||
- **Files here** (`~/.config/llamacpp/`, stowed from the repo's `lw/.config/llamacpp/`):
|
||||
`config.ini` (presets — section names ARE the API model ids), `bench.py` (CPU benchmark),
|
||||
`bench-results.md` / `bench-history.md` (ledgers, generated by bench.py). The GGUF blobs
|
||||
themselves live in `~/software/models/` and are not tracked.
|
||||
|
||||
## How it runs
|
||||
|
||||
Launched by the fish abbr **`llamaserver`** (defined in `lw/.config/fish/host.fish`;
|
||||
llama.cpp is pacman-installed, `llama-server` is on PATH):
|
||||
|
||||
```
|
||||
nice -n 19 llama-server --host 127.0.0.1 --port 11343 --cors-origins localhost \
|
||||
--models-max 1 --models-preset ~/.config/llamacpp/config.ini
|
||||
```
|
||||
|
||||
- Router mode: each requested model loads on demand in a child process.
|
||||
- **`--models-max 1`**: one model resident; a new request LRU-evicts the previous. There's no
|
||||
fl-style freeze risk here (no display-driving GPU) — this is purely the safest RAM posture on
|
||||
15 GB with no co-residency swap. Raise to 2–3 only to keep small models hot together
|
||||
(E2B+4B+1.7B ≈ 6 GB fits). `sleep-idle-seconds` per preset frees idle models either way.
|
||||
- **`nice -n 19`**: on 2 cores generation pins the whole CPU — niceness makes inference yield to
|
||||
interactive apps (full speed when the desktop is idle).
|
||||
- **`--cors-origins localhost`**: same-machine profile from llama.cpp PR 25655 — the default CORS
|
||||
reflects any Origin, so without this a malicious page in the laptop's own browser could call
|
||||
the (keyless) API on 127.0.0.1. Browser-only enforcement: bench.py/CLI clients are unaffected.
|
||||
- `config.ini` here is the stowed repo copy (`lw/.config/llamacpp/`), so editing it edits the
|
||||
repo (no deploy step).
|
||||
|
||||
## RAM (the golden rule)
|
||||
|
||||
There is no VRAM and no freeze risk. The constraint is plain: **N resident models must fit in
|
||||
system RAM together.** `bench.py` records each model's RAM footprint (MemAvailable delta) and
|
||||
the remaining free RAM — keep enough headroom for the desktop (and for co-resident models if you
|
||||
ever raise `--models-max` above 1). If RAM gets tight: shorten `sleep-idle-seconds`, use smaller
|
||||
quants, or reduce `ctx-size`. Note: the 4B/E2B presets are small enough to co-reside, but a big
|
||||
MoE (e.g. Qwen3-30B-A3B at ~12 GB) fills most of 15 GB → it runs **1-at-a-time** regardless.
|
||||
|
||||
`threads = 2` (physical cores) on the dense presets — they measure at the practical bandwidth
|
||||
wall (~8–10 GB/s effective) and the 2026-07-26 sweep confirmed more threads don't lift them.
|
||||
The **compute-bound** pair runs `threads = 3` (same-run sweep: Granite 4.5→4.8, 30B 1.9→2.0;
|
||||
`threads = 4` LOSES to HT contention on both). Every preset sets `threads-batch = 4`: prefill
|
||||
is compute-bound on all models — benched +8% prefill / −9% TTFT with no decode cost, and the
|
||||
extra HT load only bursts during prefill.
|
||||
|
||||
## Tuning cheat-sheet (CPU)
|
||||
|
||||
- **Quant size is the speed dial**: decode is bandwidth-bound, so a smaller quant is directly
|
||||
faster. Prefer Q4_K_M / Q5_K_M; F16/Q8 are usually quality-overkill AND slow on CPU.
|
||||
- **`threads`** = **physical** core count (`lscpu` → Core(s)/socket × Socket(s)). Hyperthreads
|
||||
rarely help decode; oversubscribing can hurt. Set this per preset (default 4 is a guess).
|
||||
- **Model size**: stick to ≤ ~8B. **MoE** models (few active params, e.g. Qwen3-A3B) are ideal
|
||||
IF they fit RAM — they decode at their *active* size while giving bigger-model quality. There
|
||||
is no `n-cpu-moe` tradeoff here — it's all CPU already.
|
||||
- **MTP / speculative decode** (`spec-type = draft-mtp`): a clear win on the fl GPU (+50–100%) but
|
||||
**TESTED AND REJECTED on this box** (2026-07-26): the draft lost the final CPU-pinned
|
||||
(`gpu-layers-draft = 0`) same-run A/Bs on both gemmas — E2B 4.0 vs 4.9 t/s at 47% acceptance,
|
||||
E4B 2.3 vs 2.4 at 62%. Batched verify triples the FLOPs per weight-read and 2 cores have
|
||||
no spare compute, so spec decode can't pay here at any setting. MTP stays an fl-only trick.
|
||||
(Only gemma-4 has drafts anyway; Qwen 2507/2.5 have no MTP path — that needs Qwen 3.5/3.6.)
|
||||
- **KV cache**: leave at default (f16). Quantized KV (`cache-type-*`) adds dequant overhead with
|
||||
no VRAM to reclaim → usually slower on CPU. (This is the opposite of the fl config.)
|
||||
- **iGPU / Vulkan**: the pacman build **does** have a Vulkan backend and by default auto-offloads
|
||||
layers to the HD 5500 — that's why every preset pins `n-gpu-layers = 0`. The iGPU shares the
|
||||
same DDR3, is slower than the cores, and big allocations die (`ErrorOutOfDeviceMemory`, SIGSEGV
|
||||
in llama-cli) — it broke the 30B load outright. Only re-enable offload if a bench proves the
|
||||
iGPU wins (it won't).
|
||||
|
||||
## Benchmarking
|
||||
|
||||
Run **on the laptop** (against its own localhost server):
|
||||
|
||||
```
|
||||
./bench.py # full sweep of everything the server lists
|
||||
./bench.py -m id1,id2 # just the changed presets
|
||||
./bench.py -n 512 --ctx 8000 # longer gen + a long-context decode column
|
||||
```
|
||||
|
||||
For a clean per-model RAM footprint, restart the server first (with `--models-max 1` only a
|
||||
re-bench of the *same* still-resident model reads a `~0` RAM Δ, but a restart keeps runs comparable). Results land in
|
||||
`bench-results.md` (latest) + `bench-history.md` (append-only — diff runs there).
|
||||
|
||||
⚠ Absolute numbers swing **±20–30%** with desktop load and thermals (E2B has measured 6.1 idle
|
||||
vs 4.9 warm/busy). Only rows from the **same run** are directly comparable — A/B via extra
|
||||
preset ids in one `./bench.py -m a,b` command (see the rig sections in config.ini), never by
|
||||
comparing across runs. For spec/MTP rows the draft **acceptance %** appears in the spec column.
|
||||
|
||||
## Model roster (reference numbers = clean sweep 2026-07-25, n-gpu-layers=0; Granite/30B since
|
||||
promoted to threads=3 and all presets to threads-batch=4, 2026-07-26)
|
||||
|
||||
| model id | decode t/s | prefill t/s | ctx | extras | role |
|
||||
|---|---|---|---|---|---|
|
||||
| `gemma-4-E2B-it-UD-Q4_K_XL` | **6.1** | 8 | 4k | vision-capable (mmproj) | fastest — quick Q&A, askllama-class |
|
||||
| `Qwen3-1.7B` | 4.8 | 11 | 8k | thinking | snappy small tasks |
|
||||
| `Granite-4.0-H-Tiny` | 4.3–4.8 | 11–14 | 8k | 7B/~1B-active hybrid MoE, threads 3 | ⭐ speed AND brains |
|
||||
| `Qwen3-4B-Instruct-2507` | 3.9 | 7 | 4k | non-thinking | daily driver |
|
||||
| `Qwen2.5-Coder-3B-Instruct` | 3.3 | 6 | 8k | temp 0.2 | small coder |
|
||||
| `Qwen3-30B-A3B-Instruct-2507-UD-IQ3_XXS` | 1.9–2.7 | 4 | 4k | ~12 GB, TTFT ~6 s, threads 3 | ⭐ quality when you can wait |
|
||||
| `gemma-4-E4B-it-UD-Q4_K_XL` | 2.4–2.5 | 4 | 4k | vision (mmproj) | quality vision; else the 30B is smarter at the same speed |
|
||||
| `Jan-v3-4b` | 2.1¹ | 4 | 4k | agentic tune | agentic variant of the 4B (~0.9× its speed) |
|
||||
|
||||
¹ variance-depressed in the sweep; same-run A/B vs the base (2026-07-26) shows ~0.9×.
|
||||
|
||||
## Client wiring
|
||||
|
||||
Clients run **on the laptop itself** (the server is localhost-only, see Endpoint above); same
|
||||
rule as fl otherwise: client model **id = config.ini section name**, client context ≤ server
|
||||
`ctx-size`. Point it at `http://127.0.0.1:11343/v1` (no key). If LAN access is ever needed,
|
||||
switch the abbr back to `--host 0.0.0.0` **and add `--api-key`** — never open it keyless.
|
||||
|
||||
**Wired (2026-07-26), lw-only via stow host overlays** — these are full-file `--override`
|
||||
copies of the common configs with an added local provider; when the common client configs
|
||||
change, mirror the change here:
|
||||
|
||||
| tool | overlay file | provider / client id |
|
||||
|---|---|---|
|
||||
| aichat | `lw/.config/aichat/config.yaml` | `lw:` (e.g. `lw:gemma-4-E2B-it-UD-Q4_K_XL`) |
|
||||
| opencode | `lw/.config/opencode/opencode.json` | `lwcpp/` |
|
||||
| pi | `lw/.pi/agent/models.json` + `settings.json` | `lwcpp` (⚠ pi hides providers not in settings.json `enabledModels`) |
|
||||
|
||||
All 8 presets are listed in each, with client context capped to the server `ctx-size` and the
|
||||
benched t/s in the display names. Defaults still point at the fl LAN models — switch to a local
|
||||
model in-tool when roaming. Re-run `./install.sh` on lw once so stow links the new files.
|
||||
Executable
+262
@@ -0,0 +1,262 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
llama.cpp router benchmark — LAPTOP (lw) edition → writes a results ledger for tuning.
|
||||
|
||||
This is the CPU analog of the fl (RX 7600 XT) bench.py. lw has NO discrete GPU, so all
|
||||
the ROCm / VRAM / GTT / freeze-guard / force-unload machinery from the fl version is GONE
|
||||
— on a CPU box there is no display-driving GPU to starve. The only resource ceiling is
|
||||
system RAM, so instead of VRAM this records per-model **RAM footprint** (MemAvailable
|
||||
delta, from /proc/meminfo) alongside decode/prefill/TTFT.
|
||||
|
||||
Per model it records decode/prefill t/s, TTFT, RAM used (Δ) / RAM free, and the preset's
|
||||
knobs (ctx, threads, spec-type) parsed from config.ini. For spec/MTP presets the draft
|
||||
acceptance % is appended to the spec column, read from the response's `timings` object
|
||||
(the child server also prints a `draft acceptance` log line; the table is easier to compare).
|
||||
|
||||
Output: bench-results.md (latest run, overwritten) + bench-history.md (every run, appended)
|
||||
— both next to this script. Run ON THE LAPTOP against its own server. Stdlib only.
|
||||
|
||||
./bench.py # all models the server lists
|
||||
./bench.py -m id1,id2 # only these
|
||||
./bench.py -x some-id # skip some (e.g. not downloaded yet)
|
||||
./bench.py -n 512 --ctx 8000 # longer gen + a long-context decode column
|
||||
|
||||
Note: the laptop runs `--models-max 1` (LRU eviction), so mid-sweep the RAM Δ is NET —
|
||||
each load evicts the previous model, and a smaller model after a bigger one reads negative.
|
||||
Only the first model after a server restart shows a true footprint.
|
||||
Compare a change: edit config.ini → restart server → re-run → diff runs in bench-history.md.
|
||||
"""
|
||||
import argparse, json, os, re, subprocess, sys, time, urllib.request, urllib.error
|
||||
from datetime import datetime
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
def mem_available():
|
||||
"""(available_bytes, total_bytes) from /proc/meminfo; (None, None) if unavailable."""
|
||||
try:
|
||||
info = {}
|
||||
for ln in open("/proc/meminfo", encoding="utf-8"):
|
||||
k, _, rest = ln.partition(":")
|
||||
m = re.search(r"(\d+)\s*kB", rest)
|
||||
if m:
|
||||
info[k.strip()] = int(m.group(1)) * 1024
|
||||
return info.get("MemAvailable"), info.get("MemTotal")
|
||||
except OSError:
|
||||
return None, None
|
||||
|
||||
|
||||
def with_retries(fn, retries, wait, label=""):
|
||||
"""Retry on 5xx/connection errors (model still mounting). A deterministic 'failed to
|
||||
load' from the router is NOT retried — that's a broken preset, not a slow mount."""
|
||||
last = None
|
||||
for i in range(retries + 1):
|
||||
try:
|
||||
return fn()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code < 500: # 4xx = real client error
|
||||
raise
|
||||
body = ""
|
||||
try:
|
||||
body = e.read().decode("utf-8", "ignore")
|
||||
except Exception:
|
||||
pass
|
||||
if "failed to load" in body: # deterministic → don't burn retries
|
||||
raise RuntimeError("model failed to load — check the llama-server log") from None
|
||||
last = e
|
||||
except (urllib.error.URLError, ConnectionError, TimeoutError) as e:
|
||||
last = e
|
||||
if i < retries:
|
||||
print(f" … {label} not ready ({last}); waiting {wait}s (try {i + 1}/{retries})", file=sys.stderr)
|
||||
time.sleep(wait)
|
||||
raise last
|
||||
|
||||
|
||||
def _open(url, key, payload=None, timeout=600):
|
||||
data = json.dumps(payload).encode() if payload is not None else None
|
||||
hdrs = {"Authorization": f"Bearer {key}"}
|
||||
if data:
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
return urllib.request.urlopen(urllib.request.Request(url, data=data, headers=hdrs), timeout=timeout)
|
||||
|
||||
|
||||
def list_models(base, key):
|
||||
d = json.load(_open(base.rstrip("/") + "/models", key))
|
||||
return [m["id"] for m in d.get("data", [])]
|
||||
|
||||
|
||||
def parse_config(path):
|
||||
"""{preset: {ctx, threads, spec}} from the router config.ini."""
|
||||
sections, sec = {}, None
|
||||
try:
|
||||
lines = open(path, encoding="utf-8").read().splitlines()
|
||||
except OSError:
|
||||
return {}
|
||||
for ln in lines:
|
||||
s = ln.strip()
|
||||
if s.startswith("[") and s.endswith("]"):
|
||||
sec = s[1:-1]; sections[sec] = {}
|
||||
elif sec and "=" in s and not s.startswith("#"):
|
||||
k, v = s.split("=", 1)
|
||||
sections[sec][k.strip()] = v.split("#", 1)[0].strip()
|
||||
return {sec: {"ctx": kv.get("ctx-size", "-"),
|
||||
"threads": kv.get("threads", "-"),
|
||||
"spec": kv.get("spec-type", "-") or "-"}
|
||||
for sec, kv in sections.items()}
|
||||
|
||||
|
||||
def run(base, key, model, prompt, n, temp, timeout):
|
||||
payload = {"model": model, "messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": n, "temperature": temp, "stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
# llama.cpp extension: makes the server embed its own timings in the stream →
|
||||
# the table shows the SAME predicted_per_second as the llama.cpp web UI
|
||||
# (without it we fall back to wall-clock estimates, which read lower).
|
||||
"timings_per_token": True}
|
||||
t0 = time.perf_counter(); ttft = None; ntok = 0; usage = None; timings = None
|
||||
for raw in _open(base.rstrip("/") + "/chat/completions", key, payload, timeout):
|
||||
line = raw.decode("utf-8", "ignore").strip()
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
body = line[5:].strip()
|
||||
if body == "[DONE]":
|
||||
break
|
||||
try:
|
||||
c = json.loads(body)
|
||||
except ValueError:
|
||||
continue
|
||||
ch = c.get("choices") or [{}]
|
||||
d = ch[0].get("delta", {}) if ch else {}
|
||||
# reasoning models stream `reasoning_content` first and may never emit `content`
|
||||
if d.get("content") or d.get("reasoning_content"):
|
||||
if ttft is None:
|
||||
ttft = time.perf_counter() - t0
|
||||
ntok += 1
|
||||
if c.get("usage"):
|
||||
usage = c["usage"]
|
||||
if c.get("timings"):
|
||||
timings = c["timings"]
|
||||
total = time.perf_counter() - t0
|
||||
comp = (usage or {}).get("completion_tokens") or ntok
|
||||
ptok = (usage or {}).get("prompt_tokens")
|
||||
gen_s = total - (ttft or total)
|
||||
tg = comp / gen_s if gen_s > 0 else 0.0
|
||||
pp = (ptok / ttft) if (ptok and ttft) else None
|
||||
acc = None
|
||||
if timings: # llama.cpp's own numbers are authoritative when the router forwards them
|
||||
tg = timings.get("predicted_per_second", tg)
|
||||
pp = timings.get("prompt_per_second", pp)
|
||||
ptok = timings.get("prompt_n", ptok)
|
||||
if timings.get("prompt_ms"):
|
||||
ttft = timings["prompt_ms"] / 1000
|
||||
if timings.get("draft_n"): # spec decode active — mirror the child's acceptance stat
|
||||
acc = timings.get("draft_n_accepted", 0) / timings["draft_n"]
|
||||
return {"tg": tg, "pp": pp, "ttft": ttft or 0.0, "ptok": ptok or 0, "acc": acc}
|
||||
|
||||
|
||||
def gb(b):
|
||||
return f"{b / 1e9:.1f} GB" if b else "-"
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("-u", "--url", default="http://127.0.0.1:11343/v1",
|
||||
help="run this ON the laptop against its own server (localhost)")
|
||||
ap.add_argument("-k", "--key", default="no-key-required")
|
||||
ap.add_argument("-m", "--models", default="all", help="comma-separated ids, or 'all'")
|
||||
ap.add_argument("-x", "--skip", default="", help="comma-separated ids to skip")
|
||||
ap.add_argument("-n", "--tokens", type=int, default=256)
|
||||
ap.add_argument("-t", "--temp", type=float, default=0.3)
|
||||
ap.add_argument("--ctx", type=int, default=0, help="also record decode at ~this many prompt tokens")
|
||||
ap.add_argument("--timeout", type=int, default=600, help="per-request stall timeout (s)")
|
||||
ap.add_argument("--config", default=os.path.join(HERE, "config.ini"))
|
||||
ap.add_argument("--out", default=os.path.join(HERE, "bench-results.md"))
|
||||
ap.add_argument("--history", default=os.path.join(HERE, "bench-history.md"))
|
||||
ap.add_argument("--retries", type=int, default=4, help="retries while a model is still mounting")
|
||||
ap.add_argument("--retry-wait", type=int, default=15, help="seconds between retries")
|
||||
ap.add_argument("--settle", type=float, default=1.0,
|
||||
help="seconds to wait after warmup before reading RAM (let allocation settle)")
|
||||
a = ap.parse_args()
|
||||
|
||||
models = list_models(a.url, a.key) if a.models == "all" else [m.strip() for m in a.models.split(",")]
|
||||
skip = {s.strip() for s in a.skip.split(",") if s.strip()}
|
||||
models = [m for m in models if m not in skip]
|
||||
cfg = parse_config(a.config)
|
||||
|
||||
avail0, total_ram = mem_available()
|
||||
|
||||
task = "Write a Python function that merges two sorted lists, with a short docstring and one example."
|
||||
filler = ("The quick brown fox jumps over the lazy dog. " * max(1, a.ctx // 9)) if a.ctx else None
|
||||
|
||||
cols = ["model", "cfg ctx", "threads", "spec", "decode t/s", "prefill t/s", "TTFT s", "RAM Δ", "RAM free"]
|
||||
if a.ctx:
|
||||
cols.append(f"decode@{a.ctx // 1000}k")
|
||||
|
||||
latest = open(a.out, "w", encoding="utf-8")
|
||||
hist = open(a.history, "a", encoding="utf-8")
|
||||
meta = (f"_Run {datetime.now():%Y-%m-%d %H:%M} · server `{a.url}` · RAM total {gb(total_ram)} · "
|
||||
f"baseline avail {gb(avail0)} · gen {a.tokens} tok (CPU inference, no GPU)_")
|
||||
|
||||
def emit(line, both=True):
|
||||
print(line)
|
||||
latest.write(line + "\n"); latest.flush()
|
||||
if both:
|
||||
hist.write(line + "\n"); hist.flush()
|
||||
|
||||
latest.write("# llama.cpp benchmark results — laptop (lw), latest run\n\n")
|
||||
hist.write(f"\n## run {datetime.now():%Y-%m-%d %H:%M}\n\n")
|
||||
emit(meta + "\n")
|
||||
emit("| " + " | ".join(cols) + " |")
|
||||
emit("|" + "|".join(["---"] * len(cols)) + "|")
|
||||
|
||||
for m in models:
|
||||
c = cfg.get(m, {})
|
||||
base_cells = [m, c.get("ctx", "-"), c.get("threads", "-"), c.get("spec", "-")]
|
||||
pre_avail, _ = mem_available() # per-model baseline (a model may already be resident)
|
||||
try:
|
||||
# warmup — retries wait out the mount; 'failed to load' aborts immediately
|
||||
with_retries(lambda: run(a.url, a.key, m, "hi", 8, a.temp, a.timeout), a.retries, a.retry_wait, m)
|
||||
if a.settle:
|
||||
time.sleep(a.settle)
|
||||
post_avail, _ = mem_available()
|
||||
free = post_avail
|
||||
delta = (pre_avail - post_avail) if (pre_avail is not None and post_avail is not None) else None
|
||||
# with --models-max 1 the Δ is net-of-eviction: negative = replaced a bigger model
|
||||
if delta is None:
|
||||
delta_cell = "-"
|
||||
elif delta > 0.2e9:
|
||||
delta_cell = gb(delta)
|
||||
elif delta < -0.2e9:
|
||||
delta_cell = f"-{gb(-delta)} (net swap)"
|
||||
else:
|
||||
delta_cell = "~0 (resident/net)"
|
||||
r = with_retries(lambda: run(a.url, a.key, m, task, a.tokens, a.temp, a.timeout),
|
||||
a.retries, a.retry_wait, m)
|
||||
cells = base_cells.copy()
|
||||
if r.get("acc") is not None:
|
||||
cells[3] = f"{cells[3]} ({r['acc'] * 100:.0f}% acc)"
|
||||
cells += [f"{r['tg']:.1f}", (f"{r['pp']:.0f}" if r['pp'] else "-"),
|
||||
f"{r['ttft']:.2f}", delta_cell, gb(free)]
|
||||
if a.ctx:
|
||||
rc = with_retries(lambda: run(a.url, a.key, m, filler + "\n\n" + task, a.tokens, a.temp, a.timeout),
|
||||
a.retries, a.retry_wait, m)
|
||||
cells.append(f"{rc['tg']:.1f}")
|
||||
except Exception as e:
|
||||
err = f"HTTP {e.code} (check server log)" if isinstance(e, urllib.error.HTTPError) else f"ERROR {e}"
|
||||
cells = base_cells + [err] + [""] * (len(cols) - len(base_cells) - 1)
|
||||
emit("| " + " | ".join(str(x) for x in cells) + " |")
|
||||
|
||||
latest.write("\n_Tuning hints (CPU box): decode reads the model's active bytes every token — smaller "
|
||||
"quants and small-active MoE are directly faster. **RAM free** is the real ceiling. "
|
||||
"Sanity-check: decode t/s × model GB = effective GB/s; dense models measure ~8-10 GB/s "
|
||||
"here (the practical wall), a model far below that is compute-bound (MoE/hybrid) → more "
|
||||
"`threads` may help it; `threads-batch = 4` may help prefill either way. With "
|
||||
"`--models-max 1`, mid-sweep RAM Δ is net-of-eviction. For spec/MTP models the draft "
|
||||
"acceptance % is shown in the spec column. ⚠ Absolute numbers swing ±20-30% with desktop "
|
||||
"load/thermals — only same-run rows are directly comparable._\n")
|
||||
latest.close(); hist.close()
|
||||
print(f"\nwrote {a.out} (+ appended {a.history})")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,214 @@
|
||||
# llama.cpp model config — LAPTOP (lw) · CPU inference, NO discrete GPU
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# HARDWARE: Intel Core i7-5600U (Broadwell 2015, 2 cores / 4 threads, AVX2, no AVX-512)
|
||||
# · iGPU HD Graphics 5500 (i915, not used for compute) · 15.49 GiB RAM
|
||||
# · DDR3L-1600 dual-channel ≈ ~20 GB/s effective → THE bottleneck.
|
||||
#
|
||||
# This is the laptop analog of fl/.config/llamacpp/config.ini (the RX 7600 XT box), but on a
|
||||
# CPU-only machine almost every fl rule inverts:
|
||||
#
|
||||
# • NO VRAM / GTT / freeze risk — no display-driving GPU to starve. The only ceiling is SYSTEM
|
||||
# RAM. The `llamaserver` abbr uses `--models-max 1` (one model resident; a new request LRU-
|
||||
# evicts the previous — safest on this 15 GB laptop, no co-residency swap). Raise to 2-3 only
|
||||
# to keep small models hot together (E2B+4B+1.7B ≈ 6 GB fits).
|
||||
# • Decode speed ≈ effective bandwidth ÷ active-bytes, so SMALL QUANTS are directly faster and
|
||||
# MoE (few active params) is the single biggest trick. Clean bench 2026-07-25 (n-gpu-layers=0):
|
||||
# dense models reach ~8-10 GB/s effective (decode t/s × model GB) ≈ the PRACTICAL DDR3L wall,
|
||||
# i.e. dense decode IS bandwidth-bound as designed. (The earlier "core-starved ~5-7 GB/s"
|
||||
# reading was the Vulkan-offload taint.) Granite (~2.6 GB/s eff) and the 30B (~3.5) sit far
|
||||
# BELOW the wall → COMPUTE-bound: threads 3/4 might lift those two (dense won't gain).
|
||||
# • `threads = 2` = PHYSICAL cores for the dense presets (at the wall — swept 2026-07-26, no
|
||||
# gain). `threads = 3` on the compute-bound Granite/30B: +~5-7% same-run (t4 LOSES to HT
|
||||
# contention on both — never promote it). `threads-batch = 4` EVERYWHERE: prefill is
|
||||
# compute-bound on all models — benched +8% prefill / −9% TTFT on Granite, no decode cost
|
||||
# (HT only bursts during prefill, so no sustained desktop pressure).
|
||||
# • ⚠ RESPONSIVENESS: on 2 cores, generation pins the whole CPU → the desktop can freeze. The
|
||||
# `llamaserver` abbr runs the server under `nice -n 19`, so inference YIELDS to interactive apps
|
||||
# (still full speed when the desktop is idle). Still laggy? set `threads = 1` (≈half speed, always
|
||||
# responsive).
|
||||
# • `flash-attn = on` EVERYWHERE EXCEPT the two gemma-4 presets: benched ~+11% here (gen 3.7→4.1,
|
||||
# prompt 1.7→1.9 t/s) so it's worth keeping — BUT this build (b10068) FAILS TO LOAD the gemma3n
|
||||
# arch (E2B/E4B) with flash-attn, so those two stay OFF. (If another model also won't load with
|
||||
# it, drop it there too.) No cache-type-k/v (quantized KV) on CPU: dequant overhead, no VRAM to
|
||||
# reclaim → slower. KV stays f16.
|
||||
# • ⚠ `n-gpu-layers = 0` REQUIRED in every preset — the pacman build has a VULKAN backend and
|
||||
# by default AUTO-OFFLOADS layers to the HD 5500 iGPU (common_fit_params). Seen live
|
||||
# 2026-07-25: Granite/30B loads died with "ggml_vulkan ... ErrorOutOfDeviceMemory" (llama-cli
|
||||
# even SIGSEGVs). The iGPU shares the same DDR3 and is slower than the cores — offload also
|
||||
# silently poisons benchmarks ("omit it for CPU-only" was wrong on this build).
|
||||
#
|
||||
# Clean baseline 2026-07-25 17:41 (n-gpu-layers=0, threads=2, gen 256 — bench-history.md);
|
||||
# decode noted per preset below, roster table in the README. Prefill is slow on 2 cores, so
|
||||
# huge ctx is slow to FILL (not RAM-limited) — 4096 is a sane default; raise only if needed.
|
||||
#
|
||||
# Router: llama-server --host 127.0.0.1 --port 11343 --cors-origins localhost --models-max 1 \
|
||||
# (localhost-only: the laptop roams onto untrusted networks and there's no API key —
|
||||
# unlike fl, nothing on the LAN targets this server. --cors-origins localhost blocks
|
||||
# browser pages from calling the keyless API; non-browser clients unaffected — PR 25655)
|
||||
# --models-preset ~/.config/llamacpp/config.ini (abbr `llamaserver`; llama.cpp is
|
||||
# pacman-installed → `llama-server` is on PATH)
|
||||
# This file stows to ~/.config/llamacpp/ (like fl); the GGUFs live in ~/software/models/.
|
||||
# Section names ARE the API model ids.
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Fast / small — the "instant" tier (quick Q&A, askllama, autocomplete)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
[gemma-4-E2B-it-UD-Q4_K_XL]
|
||||
# Gemma 3n E2B (~2B effective/active) — FASTEST here: benched 6.1 t/s (clean 2026-07-25). Add an mmproj for vision.
|
||||
model = /home/anon/software/models/gemma-4-E2B-it-UD-Q4_K_XL.gguf
|
||||
# flash-attn OFF: gemma3n fails to load with it on this build (b10068).
|
||||
ctx-size = 4096
|
||||
threads = 2
|
||||
threads-batch = 4
|
||||
n-gpu-layers = 0
|
||||
jinja = on
|
||||
temp = 1.0 # Gemma default (1.0 / top-p 0.95 / top-k 64); drop to 0.7 for determinism
|
||||
top-p = 0.95
|
||||
top-k = 64
|
||||
sleep-idle-seconds = 30
|
||||
|
||||
[Qwen3-1.7B]
|
||||
# Qwen3-1.7B, Q8_0 (~1.8 GB — Q8 is fine at this size). Benched 4.8 t/s (clean 2026-07-25) —
|
||||
# ~8.8 GB/s effective ≈ at the bandwidth wall; more threads won't lift this one.
|
||||
model = /home/anon/software/models/Qwen3-1.7B-Q8_0.gguf
|
||||
ctx-size = 8192 # tiny KV — can afford more ctx
|
||||
threads = 2
|
||||
threads-batch = 4
|
||||
n-gpu-layers = 0
|
||||
flash-attn = on
|
||||
jinja = on
|
||||
temp = 0.7
|
||||
top-p = 0.8
|
||||
top-k = 20
|
||||
min-p = 0
|
||||
repeat-penalty = 1.05
|
||||
sleep-idle-seconds = 30
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Daily drivers — 4B-class (best quality/speed balance; benched 2.1-3.9 t/s, clean 2026-07-25)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
[Qwen3-4B-Instruct-2507]
|
||||
# Qwen3-4B Instruct 2507, Q4_K_M (~2.5 GB). Benched 3.9 t/s (clean 2026-07-25). Best small
|
||||
# all-rounder; non-thinking, compact answers. The natural DAILY DRIVER — askllama points here.
|
||||
model = /home/anon/software/models/Qwen3-4B-Instruct-2507.Q4_K_M.gguf
|
||||
ctx-size = 4096
|
||||
threads = 2
|
||||
threads-batch = 4
|
||||
n-gpu-layers = 0
|
||||
flash-attn = on
|
||||
jinja = on
|
||||
temp = 0.7
|
||||
top-p = 0.8
|
||||
top-k = 20
|
||||
min-p = 0
|
||||
repeat-penalty = 1.05
|
||||
sleep-idle-seconds = 30
|
||||
|
||||
[gemma-4-E4B-it-UD-Q4_K_XL]
|
||||
# Gemma 3n E4B (~4B active), QAT Q4_K_XL (~4-7 GB). Quality step up from E2B; add an mmproj for vision.
|
||||
# (File is the -qat- build — QAT keeps 4-bit quality; the id drops "qat" to match fl's convention.)
|
||||
model = /home/anon/software/models/gemma-4-E4B-it-qat-UD-Q4_K_XL.gguf
|
||||
# flash-attn OFF: gemma3n fails to load with it on this build (b10068).
|
||||
# Benched 2.4 t/s (nodraft, clean 2026-07-26) — same speed as the far-smarter 30B MoE, so E4B's
|
||||
# niche is vision (mmproj). MTP draft TESTED AND REMOVED 2026-07-26: lost the final CPU-pinned
|
||||
# same-run A/Bs on both gemmas (E2B 4.0 vs 4.9 at 47% acc; E4B 2.3 vs 2.4 at 62% acc) — batched
|
||||
# verify is compute-bound on 2 cores; fl's +50% is a GPU result. The mtp-*.gguf drafts in
|
||||
# ~/software/models/ can be deleted.
|
||||
ctx-size = 4096
|
||||
threads = 2
|
||||
threads-batch = 4
|
||||
n-gpu-layers = 0
|
||||
jinja = on
|
||||
temp = 1.0
|
||||
top-p = 0.95
|
||||
top-k = 64
|
||||
sleep-idle-seconds = 30
|
||||
|
||||
[Jan-v3-4b]
|
||||
# Jan v3 4B (Qwen3-4B-based, tuned for agentic/reasoning use). Same-run A/B vs its base
|
||||
# (2026-07-26): 2.6 vs 2.9 t/s — NOT anomalous, just ~10% slower (Q4_K_XL reads a bit more per
|
||||
# token than the base's Q4_K_M). The earlier 2.1-vs-3.9 reading was cross-run variance.
|
||||
model = /home/anon/software/models/Jan-v3-4b-base-instruct-Q4_K_XL.gguf
|
||||
ctx-size = 4096 # init had 256 — unusably small for reasoning; raised
|
||||
threads = 2
|
||||
threads-batch = 4
|
||||
n-gpu-layers = 0
|
||||
flash-attn = on
|
||||
jinja = on
|
||||
temp = 0.7
|
||||
top-p = 0.95
|
||||
top-k = 20
|
||||
sleep-idle-seconds = 30
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Coding
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
[Qwen2.5-Coder-3B-Instruct]
|
||||
# Qwen2.5-Coder-3B Instruct, Q5_K_M (~2.3 GB, official Qwen GGUF). Small dedicated coder — benched 3.3 t/s (clean 2026-07-25).
|
||||
model = /home/anon/software/models/qwen2.5-coder-3b-instruct-q5_k_m.gguf
|
||||
ctx-size = 8192
|
||||
threads = 2
|
||||
threads-batch = 4
|
||||
n-gpu-layers = 0
|
||||
flash-attn = on
|
||||
jinja = on
|
||||
temp = 0.2 # low temp for code
|
||||
top-p = 0.9
|
||||
sleep-idle-seconds = 30
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# ⭐ Quality when you can wait — 30B MoE, only 3B ACTIVE → ~30B quality at ~4B speed
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
[Qwen3-30B-A3B-Instruct-2507-UD-IQ3_XXS]
|
||||
# Qwen3-30B-A3B Instruct 2507, UD-IQ3_XXS (~12 GB). Benched 1.9-2.7 t/s decode (cross-run
|
||||
# variance) / 4 t/s prefill / TTFT ~6.5 s — ~30B quality at E4B speed: THE quality pick when you
|
||||
# can wait. Compute-bound (~3.5 GB/s eff) → thread sweep 2026-07-26: t3 2.0 > t2 1.9 > t4 1.8
|
||||
# same-run → threads=3 promoted. (Earlier load failures were the Vulkan auto-offload OOM.)
|
||||
# ⚠ RAM: ~12 GB on 15.5 GB. With `--models-max 1` (the abbr default) it's always alone — no
|
||||
# co-residency swap risk — but 12 GB + desktop is still tight, so keep ctx small and close the
|
||||
# browser. If it swaps even alone, step the quant down to UD-IQ2_M (~10.5 GB).
|
||||
model = /home/anon/software/models/Qwen3-30B-A3B-Instruct-2507-UD-IQ3_XXS.gguf
|
||||
ctx-size = 4096
|
||||
threads = 3
|
||||
threads-batch = 4
|
||||
n-gpu-layers = 0
|
||||
flash-attn = on
|
||||
jinja = on
|
||||
temp = 0.7
|
||||
top-p = 0.8
|
||||
top-k = 20
|
||||
min-p = 0
|
||||
sleep-idle-seconds = 30
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# ⭐ Faster "speed AND brains" — small-active MoE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
[Granite-4.0-H-Tiny]
|
||||
# IBM Granite 4.0 H Tiny — 7B total / ~1B ACTIVE (hybrid Mamba-2 + MoE). Benched 4.3-4.8 t/s
|
||||
# decode / 11-14 t/s prefill: faster than every 4B AND ~7B-class quality, tiny RAM Δ (mmap).
|
||||
# COMPUTE-bound (~2.6 GB/s eff) → thread sweep 2026-07-26: t3 4.8 > t2 4.5 > t4 4.3 same-run →
|
||||
# threads=3 promoted (t4 loses to HT contention). Revert to 2 if the desktop drags.
|
||||
# QUANT: Q4_K_M (~4.2 GB) — goal here is speed, and Q4_K_M is the sweet spot (Q5_K_M/Q6_K fine
|
||||
# too, RAM allows, small speed cost since decode reads only the ~1B active).
|
||||
# Needs a current llama.cpp (hybrid arch) — the pacman build should be fine.
|
||||
model = /home/anon/software/models/granite-4.0-h-tiny-Q4_K_M.gguf
|
||||
ctx-size = 8192
|
||||
threads = 3
|
||||
threads-batch = 4
|
||||
n-gpu-layers = 0
|
||||
flash-attn = on # confirmed loading fine on the hybrid arch (2026-07-25)
|
||||
jinja = on
|
||||
temp = 0.7
|
||||
top-p = 0.95
|
||||
top-k = 20
|
||||
sleep-idle-seconds = 30
|
||||
|
||||
# (Bench-rig history: the MTP A/B rigs and the thread-sweep rigs that lived here were deleted
|
||||
# 2026-07-26 after their verdicts — MTP rejected (E4B comment), threads=3 promoted to
|
||||
# Granite/30B, threads-batch=4 everywhere. Raw legs in bench-history.md.)
|
||||
@@ -0,0 +1,259 @@
|
||||
{
|
||||
"$schema": "https://opencode.ai/config.json",
|
||||
"model": "llama.cpp/gemma-4-26B-A4B-it-UD-IQ4_XS",
|
||||
"small_model": "llama.cpp/gemma-4-E4B-it-UD-Q8_K_XL",
|
||||
"instructions": ["CLAUDE.md", "AGENTS.md"],
|
||||
"share": "disabled",
|
||||
"autoupdate": "notify",
|
||||
"disabled_providers": [
|
||||
"gemini",
|
||||
"anthropic",
|
||||
"groq",
|
||||
"openrouter",
|
||||
"opencode"
|
||||
],
|
||||
"permission": {
|
||||
"edit": "deny",
|
||||
"webfetch": "ask",
|
||||
"websearch": "ask",
|
||||
"external_directory": "ask",
|
||||
"read": {
|
||||
"*": "allow",
|
||||
"**/.env*": "deny",
|
||||
"**/*.env": "deny",
|
||||
"**/*.pem": "deny",
|
||||
"**/*.key": "deny",
|
||||
"**/*.p12": "deny",
|
||||
"**/*.pfx": "deny",
|
||||
"**/*.keystore": "deny",
|
||||
"**/id_rsa*": "deny",
|
||||
"**/.ssh/**": "deny",
|
||||
"**/.aws/**": "deny",
|
||||
"**/.npmrc": "deny",
|
||||
"**/.netrc": "deny"
|
||||
},
|
||||
"bash": {
|
||||
"*": "ask",
|
||||
|
||||
"ls *": "allow",
|
||||
"cat *": "allow",
|
||||
"grep *": "allow",
|
||||
"rg *": "allow",
|
||||
"find *": "allow",
|
||||
"fd *": "allow",
|
||||
"which *": "allow",
|
||||
"file *": "allow",
|
||||
"stat *": "allow",
|
||||
"wc *": "allow",
|
||||
"tree *": "allow",
|
||||
"diff *": "allow",
|
||||
"sort *": "allow",
|
||||
"uniq *": "allow",
|
||||
"echo *": "ask",
|
||||
"pwd": "allow",
|
||||
"date": "allow",
|
||||
"whoami": "allow",
|
||||
"jq *": "allow",
|
||||
"tsc *": "allow",
|
||||
"mkdir *": "ask",
|
||||
"touch *": "ask",
|
||||
|
||||
"git *": "ask",
|
||||
"git status": "allow",
|
||||
"git status *": "allow",
|
||||
"git diff": "allow",
|
||||
"git diff *": "allow",
|
||||
"git log": "allow",
|
||||
"git log *": "allow",
|
||||
"git show *": "allow",
|
||||
"git branch": "allow",
|
||||
"git remote -v": "allow",
|
||||
|
||||
"npm *": "ask",
|
||||
"pnpm *": "ask",
|
||||
"npx *": "ask",
|
||||
"npm list": "allow",
|
||||
"npm help": "allow",
|
||||
"npm version": "allow",
|
||||
"pnpm list": "allow",
|
||||
"pnpm help": "allow",
|
||||
"npx --version": "allow",
|
||||
|
||||
"curl *": "ask",
|
||||
"wget *": "ask",
|
||||
|
||||
"rm *": "deny",
|
||||
"rmdir *": "deny",
|
||||
"dd *": "deny",
|
||||
"mkfs*": "deny",
|
||||
"shred *": "deny",
|
||||
"chmod -R *": "deny",
|
||||
"sudo *": "deny",
|
||||
"doas *": "deny",
|
||||
|
||||
"npm install*": "deny",
|
||||
"npm i": "deny",
|
||||
"npm i *": "deny",
|
||||
"npm ci*": "deny",
|
||||
"pnpm install*": "deny",
|
||||
"pnpm i": "deny",
|
||||
"pnpm i *": "deny",
|
||||
"pnpm add*": "deny",
|
||||
"yarn add*": "deny",
|
||||
"yarn install*": "deny",
|
||||
"pip install*": "deny",
|
||||
"pip3 install*": "deny",
|
||||
"pipx install*": "deny",
|
||||
"cargo install*": "deny",
|
||||
"go install*": "deny",
|
||||
"gem install*": "deny",
|
||||
"brew install*": "deny",
|
||||
|
||||
"pacman *": "deny",
|
||||
"yay *": "deny",
|
||||
"apt *": "deny",
|
||||
"apt-get *": "deny",
|
||||
"dnf *": "deny",
|
||||
|
||||
"cat .env*": "deny",
|
||||
"cat *.env": "deny",
|
||||
"cat *.pem": "deny",
|
||||
"cat *.key": "deny",
|
||||
"cat *id_rsa*": "deny",
|
||||
"cat *secret*": "deny",
|
||||
"cat *credential*": "deny",
|
||||
"cat ~/.ssh/*": "deny",
|
||||
"cat ~/.aws/*": "deny",
|
||||
"grep .env*": "deny",
|
||||
"grep * .env*": "deny"
|
||||
}
|
||||
},
|
||||
"provider": {
|
||||
"lwcpp": {
|
||||
"npm": "@ai-sdk/openai-compatible",
|
||||
"name": "llamacpp (lw local)",
|
||||
"options": {
|
||||
"baseURL": "http://127.0.0.1:11343/v1"
|
||||
},
|
||||
"models": {
|
||||
"gemma-4-E2B-it-UD-Q4_K_XL": { "name": "Gemma 4 E2B · 4k — fastest on lw (~6 t/s)", "limit": { "context": 4096, "output": 1024 } },
|
||||
"Granite-4.0-H-Tiny": { "name": "Granite 4.0 H Tiny · 8k — speed AND brains (~4.8 t/s)", "limit": { "context": 8192, "output": 2048 } },
|
||||
"Qwen3-1.7B": { "name": "Qwen3 1.7B · 8k — thinking, snappy (~4.8 t/s)", "limit": { "context": 8192, "output": 2048 } },
|
||||
"Qwen3-4B-Instruct-2507": { "name": "Qwen3 4B 2507 · 4k — daily driver (~3.9 t/s)", "limit": { "context": 4096, "output": 1024 } },
|
||||
"Qwen2.5-Coder-3B-Instruct": { "name": "Qwen2.5 Coder 3B · 8k — small coder (~3.3 t/s)", "limit": { "context": 8192, "output": 2048 } },
|
||||
"Qwen3-30B-A3B-Instruct-2507-UD-IQ3_XXS": { "name": "Qwen3 30B A3B · 4k — quality when you can wait (~2.7 t/s)", "limit": { "context": 4096, "output": 1024 } },
|
||||
"Jan-v3-4b": { "name": "Jan v3 4B · 4k — agentic tune (~2.6 t/s)", "limit": { "context": 4096, "output": 1024 } },
|
||||
"gemma-4-E4B-it-UD-Q4_K_XL": { "name": "Gemma 4 E4B · 4k — quality; vision once mmproj added (~2.5 t/s)", "limit": { "context": 4096, "output": 1024 } }
|
||||
}
|
||||
},
|
||||
"duskadiy": {
|
||||
"npm": "@ai-sdk/openai-compatible",
|
||||
"name": "DuskaDIY",
|
||||
"options": {
|
||||
"baseURL": "https://llm.duskadiy.com/api/v1"
|
||||
},
|
||||
"models": {
|
||||
"Qwen3-Coder-30B-Instruct-UD-Q3_K_XL": { "name": "Qwen3 Coder 30B · 32k — main agent coder (remote)", "limit": { "context": 32768, "output": 4096 } },
|
||||
"Qwen3-Coder-Next-UD-IQ3_XXS": { "name": "Qwen3 Coder Next 80B · 128k — long sessions (remote)", "limit": { "context": 131072, "output": 16384 } },
|
||||
"Qwen3.6-35B-A3B-MTP-UD-IQ3_XXS": { "name": "Qwen3.6 35B · 24k · vision — daily driver (remote)", "attachment": true, "limit": { "context": 24576, "output": 8192 } },
|
||||
"Qwen3.6-35B-A3B-Thinking": { "name": "Qwen3.6 35B Thinking · 24k · vision — hard problems (remote)", "attachment": true, "limit": { "context": 24576, "output": 8192 } },
|
||||
"Qwen3.5-9B-UD-Q6_K_XL": { "name": "Qwen3.5 9B · 32k · vision — quick tasks (remote)", "attachment": true, "limit": { "context": 32768, "output": 8192 } },
|
||||
"gemma-4-26B-A4B-it-UD-IQ4_XS": { "name": "Gemma 4 26B · 24k · vision — quality generalist (remote)", "attachment": true, "limit": { "context": 24576, "output": 4096 } },
|
||||
"gemma-4-E4B-it-UD-Q8_K_XL": { "name": "Gemma 4 E4B · 64k · vision — fast generalist, long docs (remote)", "attachment": true, "limit": { "context": 65536, "output": 16384 } },
|
||||
"GLM-4.7-Flash-UD-Q4_K_XL": { "name": "GLM-4.7 Flash · 24k — quality coder (remote)", "limit": { "context": 24576, "output": 8192 } },
|
||||
"gpt-oss-20b": { "name": "gpt-oss 20B · 64k — fast reasoning + tools (remote)", "limit": { "context": 65536, "output": 8192 } },
|
||||
"gpt-oss-20b-low": { "name": "gpt-oss 20B low · 64k — snappy answers (remote)", "limit": { "context": 65536, "output": 8192 } }
|
||||
}
|
||||
},
|
||||
"llama.cpp": {
|
||||
"npm": "@ai-sdk/openai-compatible",
|
||||
"name": "llamacpp",
|
||||
"options": {
|
||||
"baseURL": "http://192.168.0.204:11343/v1"
|
||||
},
|
||||
"models": {
|
||||
"Qwen3-Coder-Next-UD-IQ3_XXS": { "name": "Qwen3 Coder Next 80B · 128k — long sessions", "limit": { "context": 131072, "output": 16384 } },
|
||||
"Qwen3-Coder-30B-Instruct-UD-Q3_K_XL": { "name": "Qwen3 Coder 30B · 32k — main agent coder", "limit": { "context": 32768, "output": 4096 } },
|
||||
"Qwen3.6-35B-A3B-MTP-UD-IQ3_XXS": { "name": "Qwen3.6 35B · 24k · vision — daily driver", "attachment": true, "limit": { "context": 24576, "output": 8192 } },
|
||||
"Qwen3.6-35B-A3B-Thinking": { "name": "Qwen3.6 35B Thinking · 24k · vision — hard problems", "attachment": true, "limit": { "context": 24576, "output": 8192 } },
|
||||
"Qwen3.5-9B-UD-Q6_K_XL": { "name": "Qwen3.5 9B · 32k · vision — quick tasks", "attachment": true, "limit": { "context": 32768, "output": 8192 } },
|
||||
"gemma-4-26B-A4B-it-UD-IQ4_XS": { "name": "Gemma 4 26B · 24k · vision — quality generalist", "attachment": true, "limit": { "context": 24576, "output": 4096 } },
|
||||
"gemma-4-E4B-it-UD-Q8_K_XL": { "name": "Gemma 4 E4B · 64k · vision — fast generalist, long docs", "attachment": true, "limit": { "context": 65536, "output": 16384 } },
|
||||
"GLM-4.7-Flash-UD-Q4_K_XL": { "name": "GLM-4.7 Flash · 24k — quality coder", "limit": { "context": 24576, "output": 8192 } },
|
||||
"gpt-oss-20b": { "name": "gpt-oss 20B · 64k — fast reasoning + tools", "limit": { "context": 65536, "output": 8192 } },
|
||||
"gpt-oss-20b-low": { "name": "gpt-oss 20B low · 64k — snappy answers", "limit": { "context": 65536, "output": 8192 } }
|
||||
}
|
||||
}
|
||||
},
|
||||
"agent": {
|
||||
"plan": {
|
||||
"mode": "primary",
|
||||
"model": "llama.cpp/gemma-4-26B-A4B-it-UD-IQ4_XS",
|
||||
"permission": {
|
||||
"edit": "deny"
|
||||
}
|
||||
},
|
||||
"build": {
|
||||
"mode": "primary",
|
||||
"model": "llama.cpp/gemma-4-26B-A4B-it-UD-IQ4_XS",
|
||||
"permission": {
|
||||
"edit": {
|
||||
"*": "allow",
|
||||
"**/.env*": "deny",
|
||||
"**/*.env": "deny",
|
||||
"**/*.pem": "deny",
|
||||
"**/*.key": "deny",
|
||||
"**/id_rsa*": "deny",
|
||||
"**/.ssh/**": "deny",
|
||||
"**/.aws/**": "deny",
|
||||
"**/.git/**": "deny",
|
||||
"**/.npmrc": "deny",
|
||||
"**/.netrc": "deny"
|
||||
}
|
||||
}
|
||||
},
|
||||
"code-reviewer": {
|
||||
"description": "Reviews code for best practices and potential issues",
|
||||
"prompt": "You are a code reviewer. Focus on security, performance, and maintainability.",
|
||||
"mode": "subagent",
|
||||
"model": "llama.cpp/GLM-4.7-Flash-UD-Q4_K_XL",
|
||||
"permission": {
|
||||
"edit": "deny"
|
||||
}
|
||||
},
|
||||
"test-runner": {
|
||||
"description": "Runs and interprets test suites",
|
||||
"prompt": "You are a test runner. Focus on identifying failing tests and providing clear reproduction steps.",
|
||||
"mode": "subagent",
|
||||
"model": "llama.cpp/GLM-4.7-Flash-UD-Q4_K_XL",
|
||||
"permission": {
|
||||
"edit": "deny"
|
||||
}
|
||||
},
|
||||
"doc-writer": {
|
||||
"description": "Maintains project documentation and docstrings",
|
||||
"prompt": "You are a technical writer. Focus on clarity, accuracy, and keeping documentation in sync with code changes.",
|
||||
"mode": "subagent",
|
||||
"model": "llama.cpp/gemma-4-26B-A4B-it-UD-IQ4_XS",
|
||||
"permission": {
|
||||
"edit": {
|
||||
"*": "deny",
|
||||
"**/*.md": "allow",
|
||||
"**/*.mdx": "allow",
|
||||
"**/*.markdown": "allow",
|
||||
"**/*.txt": "allow",
|
||||
"**/*.rst": "allow"
|
||||
}
|
||||
}
|
||||
},
|
||||
"security-auditor": {
|
||||
"description": "Scans for security vulnerabilities and hardcoded secrets",
|
||||
"prompt": "You are a security auditor. Focus on OWASP principles, secret detection, and dependency vulnerabilities.",
|
||||
"mode": "subagent",
|
||||
"model": "llama.cpp/GLM-4.7-Flash-UD-Q4_K_XL",
|
||||
"permission": {
|
||||
"edit": "deny"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
# F13 is a real keycode that no app binds. We declare it a virtual modifier so it can
|
||||
# prefix the nav-layer chords below, then remap RightAlt -> F13 (see modmap). This mirrors
|
||||
# the wm/fl evremap setup: RightAlt only ever emits F13, so a bare tap/hold never triggers
|
||||
# app menu bars and no stray Alt leaks on unmapped chords.
|
||||
virtual_modifiers:
|
||||
- F13
|
||||
|
||||
modmap:
|
||||
- name: CapsLock to LeftCtrl/Esc
|
||||
remap:
|
||||
CapsLock:
|
||||
held: leftctrl
|
||||
alone: esc
|
||||
alone_timeout_millis: 150
|
||||
|
||||
- name: RightAlt to F13 (nav-layer modifier; Left Alt untouched)
|
||||
remap:
|
||||
Alt_R: F13
|
||||
|
||||
keymap:
|
||||
- name: RightAlt(F13) vim-nav layer — mirrors wm/fl evremap
|
||||
remap:
|
||||
F13-h: left
|
||||
F13-j: down
|
||||
F13-k: up
|
||||
F13-l: right
|
||||
F13-u: pageup
|
||||
F13-d: pagedown
|
||||
F13-b: C-left # word left (Ctrl+Left)
|
||||
F13-w: C-right # word right (Ctrl+Right)
|
||||
|
||||
# keymap:
|
||||
# - name: Freetube remap
|
||||
# application:
|
||||
# only: FreeTube
|
||||
# remap:
|
||||
# Alt-h: Left
|
||||
# Alt-l: Right
|
||||
# Alt-j: Down
|
||||
# Alt-k: Up
|
||||
Reference in New Issue
Block a user