[Fix] cleanup

This commit is contained in:
coja
2026-08-13 01:11:41 +02:00
parent f62cb40499
commit 86726cddce
312 changed files with 12274 additions and 10772 deletions
+6
View File
@@ -0,0 +1,6 @@
## lw-specific bash extras (mirror of the fish llamaserver abbr in host.fish)
# --host 127.0.0.1 deliberately: laptop roams onto untrusted wifi and the API has no key.
# --cors-origins localhost: llama.cpp PR 25655 recommendation for same-machine use — without it
# the default CORS reflects any Origin, letting a malicious page in the local browser call the API.
alias llamaserver="nice -n 19 llama-server --host 127.0.0.1 --port 11343 --cors-origins localhost --models-max 1 --models-preset ~/.config/llamacpp/config.ini"
+82
View File
@@ -0,0 +1,82 @@
# LW HOST OVERLAY — full-file copy of common/.config/aichat/config.yaml plus the `lw` client
# (this laptop's own llama.cpp server at 127.0.0.1:11343; see lw/.config/llamacpp/README.md).
# Stowed with --override so it shadows the common file on lw only. When the common config
# changes, mirror the change here.
# see https://github.com/sigoden/aichat/blob/main/config.example.yaml
keybindings: vi
editor: nvim
model: local:Qwen3-Coder-30B-Instruct-UD-Q3_K_XL
# Sessions: persist REPL sessions and keep more history before summarizing.
# The default compress_threshold (4000) summarizes far too early for 24k+ windows.
save_session: true
compress_threshold: 16000
# REPL prompts show live context usage (needs max_input_tokens, set per model below)
left_prompt: '{color.green}{?session {session}{?role /}}{role}{color.cyan}{?rag @{rag}}{color.reset}> '
right_prompt: '{color.purple}{?session {consume_tokens}/{max_input_tokens} }{color.reset}'
# NOTE: temperature/top_p are intentionally unset — the LAN server applies tuned
# per-model sampling via --jinja (e.g. GLM 0.6/0.95); a global value would clobber it.
# max_input_tokens = real ctx-size (from the router) minus output headroom.
clients:
# This laptop's own CPU server (llamaserver abbr; works everywhere, no network needed).
# Speeds are benched decode t/s — see lw/.config/llamacpp/README.md roster.
- type: openai-compatible
name: lw
api_base: http://127.0.0.1:11343/v1
models:
- name: gemma-4-E2B-it-UD-Q4_K_XL
max_input_tokens: 3000 # ctx 4096 · ~6 t/s — fastest, quick Q&A
- name: Granite-4.0-H-Tiny
max_input_tokens: 6000 # ctx 8192 · ~4.8 t/s — speed AND brains
- name: Qwen3-1.7B
max_input_tokens: 6000 # ctx 8192 · ~4.8 t/s — thinking, snappy
- name: Qwen3-4B-Instruct-2507
max_input_tokens: 3000 # ctx 4096 · ~3.9 t/s — daily driver
- name: Qwen2.5-Coder-3B-Instruct
max_input_tokens: 6000 # ctx 8192 · ~3.3 t/s — small coder
- name: Qwen3-30B-A3B-Instruct-2507-UD-IQ3_XXS
max_input_tokens: 3000 # ctx 4096 · ~2.7 t/s — quality when you can wait
- name: Jan-v3-4b
max_input_tokens: 3000 # ctx 4096 · ~2.6 t/s — agentic tune
- name: gemma-4-E4B-it-UD-Q4_K_XL
max_input_tokens: 3000 # ctx 4096 · ~2.5 t/s — quality (vision once mmproj added)
# LAN access to the fl box (fast; only reachable on the home network)
- type: openai-compatible
name: local
api_base: http://192.168.0.204:11343/v1
models: &lan_models
- name: Qwen3-Coder-30B-Instruct-UD-Q3_K_XL
max_input_tokens: 30000 # ctx 32768
- name: Qwen3-Coder-Next-UD-IQ3_XXS
max_input_tokens: 128000 # ctx 131072
- name: Qwen3.6-35B-A3B-MTP-UD-IQ3_XXS
max_input_tokens: 22000 # ctx 24576
supports_vision: true
- name: Qwen3.6-35B-A3B-Thinking
max_input_tokens: 22000 # ctx 24576
supports_vision: true
- name: Qwen3.5-9B-UD-Q6_K_XL
max_input_tokens: 30000 # ctx 32768
supports_vision: true
- name: gemma-4-26B-A4B-it-UD-IQ4_XS
max_input_tokens: 22000 # ctx 24576
supports_vision: true
- name: gemma-4-E4B-it-UD-Q8_K_XL
max_input_tokens: 62000 # ctx 65536
supports_vision: true
- name: GLM-4.7-Flash-UD-Q4_K_XL
max_input_tokens: 22000 # ctx 24576
- name: gpt-oss-20b
max_input_tokens: 62000 # ctx 65536
- name: gpt-oss-20b-low
max_input_tokens: 62000 # ctx 65536
# Remote access via duskadiy.com (reachable from anywhere; requires an API key —
# $DUSKADIY_API_KEY, from the gitignored conf.d/secrets.fish). The LAN endpoint above is keyless.
# Same server, same model ids — reuse the list above via a YAML anchor.
- type: openai-compatible
name: duskadiy
api_base: https://llm.duskadiy.com/api/v1
models: *lan_models
+4
View File
@@ -0,0 +1,4 @@
# --host 127.0.0.1 deliberately: laptop roams onto untrusted wifi and the API has no key.
# --cors-origins localhost: llama.cpp PR 25655 recommendation for same-machine use — without it
# the default CORS reflects any Origin, letting a malicious page in the local browser call the API.
abbr llamaserver "nice -n 19 llama-server --host 127.0.0.1 --port 11343 --cors-origins localhost --models-max 1 --models-preset ~/.config/llamacpp/config.ini"
+4
View File
@@ -0,0 +1,4 @@
# lw host extras (sourced last by ~/.config/hypr/hyprland.conf)
# No discrete GPU on this laptop — no nvidia.conf.
exec-once = xremap ~/.config/xremap/config.yml # keyboard remaps (lw is the only xremap host)
+21
View File
@@ -0,0 +1,21 @@
# lw hyprlock sizes — 1920x1080, scale 1.
#
# gui/.config/hypr/hyprlock.conf ships sizes tuned for wm's 3072x1728 logical panel; these
# are those values × 0.625, which keeps the same proportions the layout was designed around
# (clock ~13% of screen height, date ~2.5%, meta ~1.5%, logo ~16%).
#
# Caveat — lw is multi-monitor and hyprlock draws on every output. monitors.conf makes
# $mainScreen (eDP-1) 1366x768, so on the built-in panel these read ~40% larger than
# intended (clock ≈ 18% of its height). Sized for the 1920x1080 externals on purpose; if
# the laptop panel is the one you actually lock on, use the 768-tuned set instead:
# $fsClock = 98 $fsDate = 20 $fsMeta = 12 $fsSmall = 11 $fsIcon = 18 $logoSize = 124
$fsClock = 138
$fsDate = 28
$fsMeta = 16
$fsSmall = 15
$fsIcon = 25
$logoSize = 175
# No $y* overrides: the shared offsets in hyprlock.conf already spread the content to the
# edges, and they land the same on both hosts because each host's sizes are the same
# proportion of its screen. Override them here if lw ever needs to differ.
+64
View File
@@ -0,0 +1,64 @@
# █▀▄▀█ █▀█ █▄░█ █ ▀█▀ █▀█ █▀█ █▀
# █░▀░█ █▄█ █░▀█ █ ░█░ █▄█ █▀▄ ▄█
# Set your monitor configuration here
# See https://wiki.hyprland.org/Configuring/Monitors/
# For a sample file, please refer to https://github.com/prasanthrangan/hyprdots/blob/main/Configs/.config/hypr/monitors.t2
$mainScreen = eDP-1
$leftScreen = DP-2
$rightScreen = HDMI-A-1
monitor = $mainScreen, 1366x768@60.14, 0x0, 1 # main screen
monitor = $leftScreen, 1920x1080@60, -1920x0, 1 # left
monitor = $rightScreen, 1920x1080@60,1366x0, 1 # right
# monitor=HDMI-A-1,1920x1080@60,-1366x0,1 # hdmi on left
# monitor=HDMI-A-1,1920x1080@60,0x-768,1 # hdmi on top
# monitor=DP-2,640x480@59.94Hz,0x0,1 # vga
# monitor = HDMI-A-1, 1366x768@60.00Hz, auto , 1, mirror, eDP-1 #mirror lapscreen
# monitor = DP-2, 1920x1080@60.00Hz, auto , 1, mirror, eDP-1 #mirror lapscreen jag3
# monitor = eDP-1, 1920x1080@60.00Hz, auto , 1, mirror, HDMI-A-1 #main hdmi, lapscreen mirrors
# monitor = eDP-1, 1920x1080@60.00Hz, auto , 1, mirror, DP-2 #main VGA, lapscreen mirrors
# monitor = , preferred, auto, 1, mirror, DP-2 # dmz projector
# monitor = eDP-1, 1920x1080@60, auto , 1, mirror, DP-2 #mirror vga test
#Main Monitor
workspace = 1, monitor:$mainScreen, default:true
workspace = 2, monitor:$mainScreen
workspace = 3, monitor:$mainScreen
workspace = 4, monitor:$mainScreen
workspace = 5, monitor:$mainScreen
workspace = 6, monitor:$mainScreen
workspace = 7, monitor:$mainScreen
workspace = 8, monitor:$mainScreen
workspace = 9, monitor:$mainScreen
workspace = 10,monitor:$mainScreen
# Second Monitor
workspace = 11, monitor:$rightScreen, default:true
workspace = 12, monitor:$rightScreen
workspace = 13, monitor:$rightScreen
workspace = 14, monitor:$rightScreen
workspace = 15, monitor:$rightScreen
workspace = 16, monitor:$rightScreen
workspace = 17, monitor:$rightScreen
workspace = 18, monitor:$rightScreen
workspace = 19, monitor:$rightScreen
workspace = 20, monitor:$rightScreen
# Third Monitor
workspace = 21, monitor:$leftScreen, default:true
workspace = 22, monitor:$leftScreen
workspace = 23, monitor:$leftScreen
workspace = 24, monitor:$leftScreen
workspace = 25, monitor:$leftScreen
workspace = 26, monitor:$leftScreen
workspace = 27, monitor:$leftScreen
workspace = 28, monitor:$leftScreen
workspace = 29, monitor:$leftScreen
workspace = 30, monitor:$leftScreen
+2
View File
@@ -0,0 +1,2 @@
# lw (laptop, smaller screen)
font_size 9.0
+13
View File
@@ -0,0 +1,13 @@
# run with `kitty --session ~/.config/kitty/music.conf`
# lw override: smaller split resizes than the shared version, to fit the laptop screen.
new_tab music
layout splits
cd ~
launch --var window=first cava
launch --location=hsplit pyradio
resize_window taller 10
focus
launch --location=vsplit pipes-rs
resize_window wider 20
+133
View File
@@ -0,0 +1,133 @@
# Local LLM server — laptop (lw), setup & usage
llama.cpp in **router mode** on the laptop. This is the **CPU** counterpart of the fl box
(`fl/.config/llamacpp/`) — lw has **no discrete GPU**, so everything runs on the CPU and
the only resource ceiling is **system RAM** (not VRAM). None of fl's VRAM/freeze rules apply.
- **Box**: Intel Core i7-5600U (Broadwell 2015, **2 cores / 4 threads**, AVX2) · iGPU HD Graphics
5500 (no compute) · **15.49 GiB RAM** · DDR3L-1600 dual-channel: ~20 GB/s theoretical, **~810
GB/s effective measured** — the dense-decode bottleneck → prefer **small quants + MoE**.
- **Endpoint**: `http://127.0.0.1:11343/v1`**local only** (server binds `127.0.0.1`; deliberate:
the laptop roams onto untrusted wifi and the API has no key, unlike the stationary fl box).
- **Files here** (`~/.config/llamacpp/`, stowed from the repo's `lw/.config/llamacpp/`):
`config.ini` (presets — section names ARE the API model ids), `bench.py` (CPU benchmark),
`bench-results.md` / `bench-history.md` (ledgers, generated by bench.py). The GGUF blobs
themselves live in `~/software/models/` and are not tracked.
## How it runs
Launched by the fish abbr **`llamaserver`** (defined in `lw/.config/fish/host.fish`;
llama.cpp is pacman-installed, `llama-server` is on PATH):
```
nice -n 19 llama-server --host 127.0.0.1 --port 11343 --cors-origins localhost \
--models-max 1 --models-preset ~/.config/llamacpp/config.ini
```
- Router mode: each requested model loads on demand in a child process.
- **`--models-max 1`**: one model resident; a new request LRU-evicts the previous. There's no
fl-style freeze risk here (no display-driving GPU) — this is purely the safest RAM posture on
15 GB with no co-residency swap. Raise to 23 only to keep small models hot together
(E2B+4B+1.7B ≈ 6 GB fits). `sleep-idle-seconds` per preset frees idle models either way.
- **`nice -n 19`**: on 2 cores generation pins the whole CPU — niceness makes inference yield to
interactive apps (full speed when the desktop is idle).
- **`--cors-origins localhost`**: same-machine profile from llama.cpp PR 25655 — the default CORS
reflects any Origin, so without this a malicious page in the laptop's own browser could call
the (keyless) API on 127.0.0.1. Browser-only enforcement: bench.py/CLI clients are unaffected.
- `config.ini` here is the stowed repo copy (`lw/.config/llamacpp/`), so editing it edits the
repo (no deploy step).
## RAM (the golden rule)
There is no VRAM and no freeze risk. The constraint is plain: **N resident models must fit in
system RAM together.** `bench.py` records each model's RAM footprint (MemAvailable delta) and
the remaining free RAM — keep enough headroom for the desktop (and for co-resident models if you
ever raise `--models-max` above 1). If RAM gets tight: shorten `sleep-idle-seconds`, use smaller
quants, or reduce `ctx-size`. Note: the 4B/E2B presets are small enough to co-reside, but a big
MoE (e.g. Qwen3-30B-A3B at ~12 GB) fills most of 15 GB → it runs **1-at-a-time** regardless.
`threads = 2` (physical cores) on the dense presets — they measure at the practical bandwidth
wall (~810 GB/s effective) and the 2026-07-26 sweep confirmed more threads don't lift them.
The **compute-bound** pair runs `threads = 3` (same-run sweep: Granite 4.5→4.8, 30B 1.9→2.0;
`threads = 4` LOSES to HT contention on both). Every preset sets `threads-batch = 4`: prefill
is compute-bound on all models — benched +8% prefill / 9% TTFT with no decode cost, and the
extra HT load only bursts during prefill.
## Tuning cheat-sheet (CPU)
- **Quant size is the speed dial**: decode is bandwidth-bound, so a smaller quant is directly
faster. Prefer Q4_K_M / Q5_K_M; F16/Q8 are usually quality-overkill AND slow on CPU.
- **`threads`** = **physical** core count (`lscpu` → Core(s)/socket × Socket(s)). Hyperthreads
rarely help decode; oversubscribing can hurt. Set this per preset (default 4 is a guess).
- **Model size**: stick to ≤ ~8B. **MoE** models (few active params, e.g. Qwen3-A3B) are ideal
IF they fit RAM — they decode at their *active* size while giving bigger-model quality. There
is no `n-cpu-moe` tradeoff here — it's all CPU already.
- **MTP / speculative decode** (`spec-type = draft-mtp`): a clear win on the fl GPU (+50100%) but
**TESTED AND REJECTED on this box** (2026-07-26): the draft lost the final CPU-pinned
(`gpu-layers-draft = 0`) same-run A/Bs on both gemmas — E2B 4.0 vs 4.9 t/s at 47% acceptance,
E4B 2.3 vs 2.4 at 62%. Batched verify triples the FLOPs per weight-read and 2 cores have
no spare compute, so spec decode can't pay here at any setting. MTP stays an fl-only trick.
(Only gemma-4 has drafts anyway; Qwen 2507/2.5 have no MTP path — that needs Qwen 3.5/3.6.)
- **KV cache**: leave at default (f16). Quantized KV (`cache-type-*`) adds dequant overhead with
no VRAM to reclaim → usually slower on CPU. (This is the opposite of the fl config.)
- **iGPU / Vulkan**: the pacman build **does** have a Vulkan backend and by default auto-offloads
layers to the HD 5500 — that's why every preset pins `n-gpu-layers = 0`. The iGPU shares the
same DDR3, is slower than the cores, and big allocations die (`ErrorOutOfDeviceMemory`, SIGSEGV
in llama-cli) — it broke the 30B load outright. Only re-enable offload if a bench proves the
iGPU wins (it won't).
## Benchmarking
Run **on the laptop** (against its own localhost server):
```
./bench.py # full sweep of everything the server lists
./bench.py -m id1,id2 # just the changed presets
./bench.py -n 512 --ctx 8000 # longer gen + a long-context decode column
```
For a clean per-model RAM footprint, restart the server first (with `--models-max 1` only a
re-bench of the *same* still-resident model reads a `~0` RAM Δ, but a restart keeps runs comparable). Results land in
`bench-results.md` (latest) + `bench-history.md` (append-only — diff runs there).
⚠ Absolute numbers swing **±2030%** with desktop load and thermals (E2B has measured 6.1 idle
vs 4.9 warm/busy). Only rows from the **same run** are directly comparable — A/B via extra
preset ids in one `./bench.py -m a,b` command (see the rig sections in config.ini), never by
comparing across runs. For spec/MTP rows the draft **acceptance %** appears in the spec column.
## Model roster (reference numbers = clean sweep 2026-07-25, n-gpu-layers=0; Granite/30B since
promoted to threads=3 and all presets to threads-batch=4, 2026-07-26)
| model id | decode t/s | prefill t/s | ctx | extras | role |
|---|---|---|---|---|---|
| `gemma-4-E2B-it-UD-Q4_K_XL` | **6.1** | 8 | 4k | vision-capable (mmproj) | fastest — quick Q&A, askllama-class |
| `Qwen3-1.7B` | 4.8 | 11 | 8k | thinking | snappy small tasks |
| `Granite-4.0-H-Tiny` | 4.34.8 | 1114 | 8k | 7B/~1B-active hybrid MoE, threads 3 | ⭐ speed AND brains |
| `Qwen3-4B-Instruct-2507` | 3.9 | 7 | 4k | non-thinking | daily driver |
| `Qwen2.5-Coder-3B-Instruct` | 3.3 | 6 | 8k | temp 0.2 | small coder |
| `Qwen3-30B-A3B-Instruct-2507-UD-IQ3_XXS` | 1.92.7 | 4 | 4k | ~12 GB, TTFT ~6 s, threads 3 | ⭐ quality when you can wait |
| `gemma-4-E4B-it-UD-Q4_K_XL` | 2.42.5 | 4 | 4k | vision (mmproj) | quality vision; else the 30B is smarter at the same speed |
| `Jan-v3-4b` | 2.1¹ | 4 | 4k | agentic tune | agentic variant of the 4B (~0.9× its speed) |
¹ variance-depressed in the sweep; same-run A/B vs the base (2026-07-26) shows ~0.9×.
## Client wiring
Clients run **on the laptop itself** (the server is localhost-only, see Endpoint above); same
rule as fl otherwise: client model **id = config.ini section name**, client context ≤ server
`ctx-size`. Point it at `http://127.0.0.1:11343/v1` (no key). If LAN access is ever needed,
switch the abbr back to `--host 0.0.0.0` **and add `--api-key`** — never open it keyless.
**Wired (2026-07-26), lw-only via stow host overlays** — these are full-file `--override`
copies of the common configs with an added local provider; when the common client configs
change, mirror the change here:
| tool | overlay file | provider / client id |
|---|---|---|
| aichat | `lw/.config/aichat/config.yaml` | `lw:` (e.g. `lw:gemma-4-E2B-it-UD-Q4_K_XL`) |
| opencode | `lw/.config/opencode/opencode.json` | `lwcpp/` |
| pi | `lw/.pi/agent/models.json` + `settings.json` | `lwcpp` (⚠ pi hides providers not in settings.json `enabledModels`) |
All 8 presets are listed in each, with client context capped to the server `ctx-size` and the
benched t/s in the display names. Defaults still point at the fl LAN models — switch to a local
model in-tool when roaming. Re-run `./install.sh` on lw once so stow links the new files.
+262
View File
@@ -0,0 +1,262 @@
#!/usr/bin/env python3
"""
llama.cpp router benchmark — LAPTOP (lw) edition → writes a results ledger for tuning.
This is the CPU analog of the fl (RX 7600 XT) bench.py. lw has NO discrete GPU, so all
the ROCm / VRAM / GTT / freeze-guard / force-unload machinery from the fl version is GONE
— on a CPU box there is no display-driving GPU to starve. The only resource ceiling is
system RAM, so instead of VRAM this records per-model **RAM footprint** (MemAvailable
delta, from /proc/meminfo) alongside decode/prefill/TTFT.
Per model it records decode/prefill t/s, TTFT, RAM used (Δ) / RAM free, and the preset's
knobs (ctx, threads, spec-type) parsed from config.ini. For spec/MTP presets the draft
acceptance % is appended to the spec column, read from the response's `timings` object
(the child server also prints a `draft acceptance` log line; the table is easier to compare).
Output: bench-results.md (latest run, overwritten) + bench-history.md (every run, appended)
— both next to this script. Run ON THE LAPTOP against its own server. Stdlib only.
./bench.py # all models the server lists
./bench.py -m id1,id2 # only these
./bench.py -x some-id # skip some (e.g. not downloaded yet)
./bench.py -n 512 --ctx 8000 # longer gen + a long-context decode column
Note: the laptop runs `--models-max 1` (LRU eviction), so mid-sweep the RAM Δ is NET —
each load evicts the previous model, and a smaller model after a bigger one reads negative.
Only the first model after a server restart shows a true footprint.
Compare a change: edit config.ini → restart server → re-run → diff runs in bench-history.md.
"""
import argparse, json, os, re, subprocess, sys, time, urllib.request, urllib.error
from datetime import datetime
HERE = os.path.dirname(os.path.abspath(__file__))
def mem_available():
"""(available_bytes, total_bytes) from /proc/meminfo; (None, None) if unavailable."""
try:
info = {}
for ln in open("/proc/meminfo", encoding="utf-8"):
k, _, rest = ln.partition(":")
m = re.search(r"(\d+)\s*kB", rest)
if m:
info[k.strip()] = int(m.group(1)) * 1024
return info.get("MemAvailable"), info.get("MemTotal")
except OSError:
return None, None
def with_retries(fn, retries, wait, label=""):
"""Retry on 5xx/connection errors (model still mounting). A deterministic 'failed to
load' from the router is NOT retried — that's a broken preset, not a slow mount."""
last = None
for i in range(retries + 1):
try:
return fn()
except urllib.error.HTTPError as e:
if e.code < 500: # 4xx = real client error
raise
body = ""
try:
body = e.read().decode("utf-8", "ignore")
except Exception:
pass
if "failed to load" in body: # deterministic → don't burn retries
raise RuntimeError("model failed to load — check the llama-server log") from None
last = e
except (urllib.error.URLError, ConnectionError, TimeoutError) as e:
last = e
if i < retries:
print(f"{label} not ready ({last}); waiting {wait}s (try {i + 1}/{retries})", file=sys.stderr)
time.sleep(wait)
raise last
def _open(url, key, payload=None, timeout=600):
data = json.dumps(payload).encode() if payload is not None else None
hdrs = {"Authorization": f"Bearer {key}"}
if data:
hdrs["Content-Type"] = "application/json"
return urllib.request.urlopen(urllib.request.Request(url, data=data, headers=hdrs), timeout=timeout)
def list_models(base, key):
d = json.load(_open(base.rstrip("/") + "/models", key))
return [m["id"] for m in d.get("data", [])]
def parse_config(path):
"""{preset: {ctx, threads, spec}} from the router config.ini."""
sections, sec = {}, None
try:
lines = open(path, encoding="utf-8").read().splitlines()
except OSError:
return {}
for ln in lines:
s = ln.strip()
if s.startswith("[") and s.endswith("]"):
sec = s[1:-1]; sections[sec] = {}
elif sec and "=" in s and not s.startswith("#"):
k, v = s.split("=", 1)
sections[sec][k.strip()] = v.split("#", 1)[0].strip()
return {sec: {"ctx": kv.get("ctx-size", "-"),
"threads": kv.get("threads", "-"),
"spec": kv.get("spec-type", "-") or "-"}
for sec, kv in sections.items()}
def run(base, key, model, prompt, n, temp, timeout):
payload = {"model": model, "messages": [{"role": "user", "content": prompt}],
"max_tokens": n, "temperature": temp, "stream": True,
"stream_options": {"include_usage": True},
# llama.cpp extension: makes the server embed its own timings in the stream →
# the table shows the SAME predicted_per_second as the llama.cpp web UI
# (without it we fall back to wall-clock estimates, which read lower).
"timings_per_token": True}
t0 = time.perf_counter(); ttft = None; ntok = 0; usage = None; timings = None
for raw in _open(base.rstrip("/") + "/chat/completions", key, payload, timeout):
line = raw.decode("utf-8", "ignore").strip()
if not line.startswith("data:"):
continue
body = line[5:].strip()
if body == "[DONE]":
break
try:
c = json.loads(body)
except ValueError:
continue
ch = c.get("choices") or [{}]
d = ch[0].get("delta", {}) if ch else {}
# reasoning models stream `reasoning_content` first and may never emit `content`
if d.get("content") or d.get("reasoning_content"):
if ttft is None:
ttft = time.perf_counter() - t0
ntok += 1
if c.get("usage"):
usage = c["usage"]
if c.get("timings"):
timings = c["timings"]
total = time.perf_counter() - t0
comp = (usage or {}).get("completion_tokens") or ntok
ptok = (usage or {}).get("prompt_tokens")
gen_s = total - (ttft or total)
tg = comp / gen_s if gen_s > 0 else 0.0
pp = (ptok / ttft) if (ptok and ttft) else None
acc = None
if timings: # llama.cpp's own numbers are authoritative when the router forwards them
tg = timings.get("predicted_per_second", tg)
pp = timings.get("prompt_per_second", pp)
ptok = timings.get("prompt_n", ptok)
if timings.get("prompt_ms"):
ttft = timings["prompt_ms"] / 1000
if timings.get("draft_n"): # spec decode active — mirror the child's acceptance stat
acc = timings.get("draft_n_accepted", 0) / timings["draft_n"]
return {"tg": tg, "pp": pp, "ttft": ttft or 0.0, "ptok": ptok or 0, "acc": acc}
def gb(b):
return f"{b / 1e9:.1f} GB" if b else "-"
def main():
ap = argparse.ArgumentParser()
ap.add_argument("-u", "--url", default="http://127.0.0.1:11343/v1",
help="run this ON the laptop against its own server (localhost)")
ap.add_argument("-k", "--key", default="no-key-required")
ap.add_argument("-m", "--models", default="all", help="comma-separated ids, or 'all'")
ap.add_argument("-x", "--skip", default="", help="comma-separated ids to skip")
ap.add_argument("-n", "--tokens", type=int, default=256)
ap.add_argument("-t", "--temp", type=float, default=0.3)
ap.add_argument("--ctx", type=int, default=0, help="also record decode at ~this many prompt tokens")
ap.add_argument("--timeout", type=int, default=600, help="per-request stall timeout (s)")
ap.add_argument("--config", default=os.path.join(HERE, "config.ini"))
ap.add_argument("--out", default=os.path.join(HERE, "bench-results.md"))
ap.add_argument("--history", default=os.path.join(HERE, "bench-history.md"))
ap.add_argument("--retries", type=int, default=4, help="retries while a model is still mounting")
ap.add_argument("--retry-wait", type=int, default=15, help="seconds between retries")
ap.add_argument("--settle", type=float, default=1.0,
help="seconds to wait after warmup before reading RAM (let allocation settle)")
a = ap.parse_args()
models = list_models(a.url, a.key) if a.models == "all" else [m.strip() for m in a.models.split(",")]
skip = {s.strip() for s in a.skip.split(",") if s.strip()}
models = [m for m in models if m not in skip]
cfg = parse_config(a.config)
avail0, total_ram = mem_available()
task = "Write a Python function that merges two sorted lists, with a short docstring and one example."
filler = ("The quick brown fox jumps over the lazy dog. " * max(1, a.ctx // 9)) if a.ctx else None
cols = ["model", "cfg ctx", "threads", "spec", "decode t/s", "prefill t/s", "TTFT s", "RAM Δ", "RAM free"]
if a.ctx:
cols.append(f"decode@{a.ctx // 1000}k")
latest = open(a.out, "w", encoding="utf-8")
hist = open(a.history, "a", encoding="utf-8")
meta = (f"_Run {datetime.now():%Y-%m-%d %H:%M} · server `{a.url}` · RAM total {gb(total_ram)} · "
f"baseline avail {gb(avail0)} · gen {a.tokens} tok (CPU inference, no GPU)_")
def emit(line, both=True):
print(line)
latest.write(line + "\n"); latest.flush()
if both:
hist.write(line + "\n"); hist.flush()
latest.write("# llama.cpp benchmark results — laptop (lw), latest run\n\n")
hist.write(f"\n## run {datetime.now():%Y-%m-%d %H:%M}\n\n")
emit(meta + "\n")
emit("| " + " | ".join(cols) + " |")
emit("|" + "|".join(["---"] * len(cols)) + "|")
for m in models:
c = cfg.get(m, {})
base_cells = [m, c.get("ctx", "-"), c.get("threads", "-"), c.get("spec", "-")]
pre_avail, _ = mem_available() # per-model baseline (a model may already be resident)
try:
# warmup — retries wait out the mount; 'failed to load' aborts immediately
with_retries(lambda: run(a.url, a.key, m, "hi", 8, a.temp, a.timeout), a.retries, a.retry_wait, m)
if a.settle:
time.sleep(a.settle)
post_avail, _ = mem_available()
free = post_avail
delta = (pre_avail - post_avail) if (pre_avail is not None and post_avail is not None) else None
# with --models-max 1 the Δ is net-of-eviction: negative = replaced a bigger model
if delta is None:
delta_cell = "-"
elif delta > 0.2e9:
delta_cell = gb(delta)
elif delta < -0.2e9:
delta_cell = f"-{gb(-delta)} (net swap)"
else:
delta_cell = "~0 (resident/net)"
r = with_retries(lambda: run(a.url, a.key, m, task, a.tokens, a.temp, a.timeout),
a.retries, a.retry_wait, m)
cells = base_cells.copy()
if r.get("acc") is not None:
cells[3] = f"{cells[3]} ({r['acc'] * 100:.0f}% acc)"
cells += [f"{r['tg']:.1f}", (f"{r['pp']:.0f}" if r['pp'] else "-"),
f"{r['ttft']:.2f}", delta_cell, gb(free)]
if a.ctx:
rc = with_retries(lambda: run(a.url, a.key, m, filler + "\n\n" + task, a.tokens, a.temp, a.timeout),
a.retries, a.retry_wait, m)
cells.append(f"{rc['tg']:.1f}")
except Exception as e:
err = f"HTTP {e.code} (check server log)" if isinstance(e, urllib.error.HTTPError) else f"ERROR {e}"
cells = base_cells + [err] + [""] * (len(cols) - len(base_cells) - 1)
emit("| " + " | ".join(str(x) for x in cells) + " |")
latest.write("\n_Tuning hints (CPU box): decode reads the model's active bytes every token — smaller "
"quants and small-active MoE are directly faster. **RAM free** is the real ceiling. "
"Sanity-check: decode t/s × model GB = effective GB/s; dense models measure ~8-10 GB/s "
"here (the practical wall), a model far below that is compute-bound (MoE/hybrid) → more "
"`threads` may help it; `threads-batch = 4` may help prefill either way. With "
"`--models-max 1`, mid-sweep RAM Δ is net-of-eviction. For spec/MTP models the draft "
"acceptance % is shown in the spec column. ⚠ Absolute numbers swing ±20-30% with desktop "
"load/thermals — only same-run rows are directly comparable._\n")
latest.close(); hist.close()
print(f"\nwrote {a.out} (+ appended {a.history})")
if __name__ == "__main__":
main()
+214
View File
@@ -0,0 +1,214 @@
# llama.cpp model config — LAPTOP (lw) · CPU inference, NO discrete GPU
# ═════════════════════════════════════════════════════════════════════════════
# HARDWARE: Intel Core i7-5600U (Broadwell 2015, 2 cores / 4 threads, AVX2, no AVX-512)
# · iGPU HD Graphics 5500 (i915, not used for compute) · 15.49 GiB RAM
# · DDR3L-1600 dual-channel ≈ ~20 GB/s effective → THE bottleneck.
#
# This is the laptop analog of fl/.config/llamacpp/config.ini (the RX 7600 XT box), but on a
# CPU-only machine almost every fl rule inverts:
#
# • NO VRAM / GTT / freeze risk — no display-driving GPU to starve. The only ceiling is SYSTEM
# RAM. The `llamaserver` abbr uses `--models-max 1` (one model resident; a new request LRU-
# evicts the previous — safest on this 15 GB laptop, no co-residency swap). Raise to 2-3 only
# to keep small models hot together (E2B+4B+1.7B ≈ 6 GB fits).
# • Decode speed ≈ effective bandwidth ÷ active-bytes, so SMALL QUANTS are directly faster and
# MoE (few active params) is the single biggest trick. Clean bench 2026-07-25 (n-gpu-layers=0):
# dense models reach ~8-10 GB/s effective (decode t/s × model GB) ≈ the PRACTICAL DDR3L wall,
# i.e. dense decode IS bandwidth-bound as designed. (The earlier "core-starved ~5-7 GB/s"
# reading was the Vulkan-offload taint.) Granite (~2.6 GB/s eff) and the 30B (~3.5) sit far
# BELOW the wall → COMPUTE-bound: threads 3/4 might lift those two (dense won't gain).
# • `threads = 2` = PHYSICAL cores for the dense presets (at the wall — swept 2026-07-26, no
# gain). `threads = 3` on the compute-bound Granite/30B: +~5-7% same-run (t4 LOSES to HT
# contention on both — never promote it). `threads-batch = 4` EVERYWHERE: prefill is
# compute-bound on all models — benched +8% prefill / 9% TTFT on Granite, no decode cost
# (HT only bursts during prefill, so no sustained desktop pressure).
# • ⚠ RESPONSIVENESS: on 2 cores, generation pins the whole CPU → the desktop can freeze. The
# `llamaserver` abbr runs the server under `nice -n 19`, so inference YIELDS to interactive apps
# (still full speed when the desktop is idle). Still laggy? set `threads = 1` (≈half speed, always
# responsive).
# • `flash-attn = on` EVERYWHERE EXCEPT the two gemma-4 presets: benched ~+11% here (gen 3.7→4.1,
# prompt 1.7→1.9 t/s) so it's worth keeping — BUT this build (b10068) FAILS TO LOAD the gemma3n
# arch (E2B/E4B) with flash-attn, so those two stay OFF. (If another model also won't load with
# it, drop it there too.) No cache-type-k/v (quantized KV) on CPU: dequant overhead, no VRAM to
# reclaim → slower. KV stays f16.
# • ⚠ `n-gpu-layers = 0` REQUIRED in every preset — the pacman build has a VULKAN backend and
# by default AUTO-OFFLOADS layers to the HD 5500 iGPU (common_fit_params). Seen live
# 2026-07-25: Granite/30B loads died with "ggml_vulkan ... ErrorOutOfDeviceMemory" (llama-cli
# even SIGSEGVs). The iGPU shares the same DDR3 and is slower than the cores — offload also
# silently poisons benchmarks ("omit it for CPU-only" was wrong on this build).
#
# Clean baseline 2026-07-25 17:41 (n-gpu-layers=0, threads=2, gen 256 — bench-history.md);
# decode noted per preset below, roster table in the README. Prefill is slow on 2 cores, so
# huge ctx is slow to FILL (not RAM-limited) — 4096 is a sane default; raise only if needed.
#
# Router: llama-server --host 127.0.0.1 --port 11343 --cors-origins localhost --models-max 1 \
# (localhost-only: the laptop roams onto untrusted networks and there's no API key —
# unlike fl, nothing on the LAN targets this server. --cors-origins localhost blocks
# browser pages from calling the keyless API; non-browser clients unaffected — PR 25655)
# --models-preset ~/.config/llamacpp/config.ini (abbr `llamaserver`; llama.cpp is
# pacman-installed → `llama-server` is on PATH)
# This file stows to ~/.config/llamacpp/ (like fl); the GGUFs live in ~/software/models/.
# Section names ARE the API model ids.
# ═════════════════════════════════════════════════════════════════════════════
# ─────────────────────────────────────────────────────────────────────────────
# Fast / small — the "instant" tier (quick Q&A, askllama, autocomplete)
# ─────────────────────────────────────────────────────────────────────────────
[gemma-4-E2B-it-UD-Q4_K_XL]
# Gemma 3n E2B (~2B effective/active) — FASTEST here: benched 6.1 t/s (clean 2026-07-25). Add an mmproj for vision.
model = /home/anon/software/models/gemma-4-E2B-it-UD-Q4_K_XL.gguf
# flash-attn OFF: gemma3n fails to load with it on this build (b10068).
ctx-size = 4096
threads = 2
threads-batch = 4
n-gpu-layers = 0
jinja = on
temp = 1.0 # Gemma default (1.0 / top-p 0.95 / top-k 64); drop to 0.7 for determinism
top-p = 0.95
top-k = 64
sleep-idle-seconds = 30
[Qwen3-1.7B]
# Qwen3-1.7B, Q8_0 (~1.8 GB — Q8 is fine at this size). Benched 4.8 t/s (clean 2026-07-25) —
# ~8.8 GB/s effective ≈ at the bandwidth wall; more threads won't lift this one.
model = /home/anon/software/models/Qwen3-1.7B-Q8_0.gguf
ctx-size = 8192 # tiny KV — can afford more ctx
threads = 2
threads-batch = 4
n-gpu-layers = 0
flash-attn = on
jinja = on
temp = 0.7
top-p = 0.8
top-k = 20
min-p = 0
repeat-penalty = 1.05
sleep-idle-seconds = 30
# ─────────────────────────────────────────────────────────────────────────────
# Daily drivers — 4B-class (best quality/speed balance; benched 2.1-3.9 t/s, clean 2026-07-25)
# ─────────────────────────────────────────────────────────────────────────────
[Qwen3-4B-Instruct-2507]
# Qwen3-4B Instruct 2507, Q4_K_M (~2.5 GB). Benched 3.9 t/s (clean 2026-07-25). Best small
# all-rounder; non-thinking, compact answers. The natural DAILY DRIVER — askllama points here.
model = /home/anon/software/models/Qwen3-4B-Instruct-2507.Q4_K_M.gguf
ctx-size = 4096
threads = 2
threads-batch = 4
n-gpu-layers = 0
flash-attn = on
jinja = on
temp = 0.7
top-p = 0.8
top-k = 20
min-p = 0
repeat-penalty = 1.05
sleep-idle-seconds = 30
[gemma-4-E4B-it-UD-Q4_K_XL]
# Gemma 3n E4B (~4B active), QAT Q4_K_XL (~4-7 GB). Quality step up from E2B; add an mmproj for vision.
# (File is the -qat- build — QAT keeps 4-bit quality; the id drops "qat" to match fl's convention.)
model = /home/anon/software/models/gemma-4-E4B-it-qat-UD-Q4_K_XL.gguf
# flash-attn OFF: gemma3n fails to load with it on this build (b10068).
# Benched 2.4 t/s (nodraft, clean 2026-07-26) — same speed as the far-smarter 30B MoE, so E4B's
# niche is vision (mmproj). MTP draft TESTED AND REMOVED 2026-07-26: lost the final CPU-pinned
# same-run A/Bs on both gemmas (E2B 4.0 vs 4.9 at 47% acc; E4B 2.3 vs 2.4 at 62% acc) — batched
# verify is compute-bound on 2 cores; fl's +50% is a GPU result. The mtp-*.gguf drafts in
# ~/software/models/ can be deleted.
ctx-size = 4096
threads = 2
threads-batch = 4
n-gpu-layers = 0
jinja = on
temp = 1.0
top-p = 0.95
top-k = 64
sleep-idle-seconds = 30
[Jan-v3-4b]
# Jan v3 4B (Qwen3-4B-based, tuned for agentic/reasoning use). Same-run A/B vs its base
# (2026-07-26): 2.6 vs 2.9 t/s — NOT anomalous, just ~10% slower (Q4_K_XL reads a bit more per
# token than the base's Q4_K_M). The earlier 2.1-vs-3.9 reading was cross-run variance.
model = /home/anon/software/models/Jan-v3-4b-base-instruct-Q4_K_XL.gguf
ctx-size = 4096 # init had 256 — unusably small for reasoning; raised
threads = 2
threads-batch = 4
n-gpu-layers = 0
flash-attn = on
jinja = on
temp = 0.7
top-p = 0.95
top-k = 20
sleep-idle-seconds = 30
# ─────────────────────────────────────────────────────────────────────────────
# Coding
# ─────────────────────────────────────────────────────────────────────────────
[Qwen2.5-Coder-3B-Instruct]
# Qwen2.5-Coder-3B Instruct, Q5_K_M (~2.3 GB, official Qwen GGUF). Small dedicated coder — benched 3.3 t/s (clean 2026-07-25).
model = /home/anon/software/models/qwen2.5-coder-3b-instruct-q5_k_m.gguf
ctx-size = 8192
threads = 2
threads-batch = 4
n-gpu-layers = 0
flash-attn = on
jinja = on
temp = 0.2 # low temp for code
top-p = 0.9
sleep-idle-seconds = 30
# ─────────────────────────────────────────────────────────────────────────────
# ⭐ Quality when you can wait — 30B MoE, only 3B ACTIVE → ~30B quality at ~4B speed
# ─────────────────────────────────────────────────────────────────────────────
[Qwen3-30B-A3B-Instruct-2507-UD-IQ3_XXS]
# Qwen3-30B-A3B Instruct 2507, UD-IQ3_XXS (~12 GB). Benched 1.9-2.7 t/s decode (cross-run
# variance) / 4 t/s prefill / TTFT ~6.5 s — ~30B quality at E4B speed: THE quality pick when you
# can wait. Compute-bound (~3.5 GB/s eff) → thread sweep 2026-07-26: t3 2.0 > t2 1.9 > t4 1.8
# same-run → threads=3 promoted. (Earlier load failures were the Vulkan auto-offload OOM.)
# ⚠ RAM: ~12 GB on 15.5 GB. With `--models-max 1` (the abbr default) it's always alone — no
# co-residency swap risk — but 12 GB + desktop is still tight, so keep ctx small and close the
# browser. If it swaps even alone, step the quant down to UD-IQ2_M (~10.5 GB).
model = /home/anon/software/models/Qwen3-30B-A3B-Instruct-2507-UD-IQ3_XXS.gguf
ctx-size = 4096
threads = 3
threads-batch = 4
n-gpu-layers = 0
flash-attn = on
jinja = on
temp = 0.7
top-p = 0.8
top-k = 20
min-p = 0
sleep-idle-seconds = 30
# ─────────────────────────────────────────────────────────────────────────────
# ⭐ Faster "speed AND brains" — small-active MoE
# ─────────────────────────────────────────────────────────────────────────────
[Granite-4.0-H-Tiny]
# IBM Granite 4.0 H Tiny — 7B total / ~1B ACTIVE (hybrid Mamba-2 + MoE). Benched 4.3-4.8 t/s
# decode / 11-14 t/s prefill: faster than every 4B AND ~7B-class quality, tiny RAM Δ (mmap).
# COMPUTE-bound (~2.6 GB/s eff) → thread sweep 2026-07-26: t3 4.8 > t2 4.5 > t4 4.3 same-run →
# threads=3 promoted (t4 loses to HT contention). Revert to 2 if the desktop drags.
# QUANT: Q4_K_M (~4.2 GB) — goal here is speed, and Q4_K_M is the sweet spot (Q5_K_M/Q6_K fine
# too, RAM allows, small speed cost since decode reads only the ~1B active).
# Needs a current llama.cpp (hybrid arch) — the pacman build should be fine.
model = /home/anon/software/models/granite-4.0-h-tiny-Q4_K_M.gguf
ctx-size = 8192
threads = 3
threads-batch = 4
n-gpu-layers = 0
flash-attn = on # confirmed loading fine on the hybrid arch (2026-07-25)
jinja = on
temp = 0.7
top-p = 0.95
top-k = 20
sleep-idle-seconds = 30
# (Bench-rig history: the MTP A/B rigs and the thread-sweep rigs that lived here were deleted
# 2026-07-26 after their verdicts — MTP rejected (E4B comment), threads=3 promoted to
# Granite/30B, threads-batch=4 everywhere. Raw legs in bench-history.md.)
+259
View File
@@ -0,0 +1,259 @@
{
"$schema": "https://opencode.ai/config.json",
"model": "llama.cpp/gemma-4-26B-A4B-it-UD-IQ4_XS",
"small_model": "llama.cpp/gemma-4-E4B-it-UD-Q8_K_XL",
"instructions": ["CLAUDE.md", "AGENTS.md"],
"share": "disabled",
"autoupdate": "notify",
"disabled_providers": [
"gemini",
"anthropic",
"groq",
"openrouter",
"opencode"
],
"permission": {
"edit": "deny",
"webfetch": "ask",
"websearch": "ask",
"external_directory": "ask",
"read": {
"*": "allow",
"**/.env*": "deny",
"**/*.env": "deny",
"**/*.pem": "deny",
"**/*.key": "deny",
"**/*.p12": "deny",
"**/*.pfx": "deny",
"**/*.keystore": "deny",
"**/id_rsa*": "deny",
"**/.ssh/**": "deny",
"**/.aws/**": "deny",
"**/.npmrc": "deny",
"**/.netrc": "deny"
},
"bash": {
"*": "ask",
"ls *": "allow",
"cat *": "allow",
"grep *": "allow",
"rg *": "allow",
"find *": "allow",
"fd *": "allow",
"which *": "allow",
"file *": "allow",
"stat *": "allow",
"wc *": "allow",
"tree *": "allow",
"diff *": "allow",
"sort *": "allow",
"uniq *": "allow",
"echo *": "ask",
"pwd": "allow",
"date": "allow",
"whoami": "allow",
"jq *": "allow",
"tsc *": "allow",
"mkdir *": "ask",
"touch *": "ask",
"git *": "ask",
"git status": "allow",
"git status *": "allow",
"git diff": "allow",
"git diff *": "allow",
"git log": "allow",
"git log *": "allow",
"git show *": "allow",
"git branch": "allow",
"git remote -v": "allow",
"npm *": "ask",
"pnpm *": "ask",
"npx *": "ask",
"npm list": "allow",
"npm help": "allow",
"npm version": "allow",
"pnpm list": "allow",
"pnpm help": "allow",
"npx --version": "allow",
"curl *": "ask",
"wget *": "ask",
"rm *": "deny",
"rmdir *": "deny",
"dd *": "deny",
"mkfs*": "deny",
"shred *": "deny",
"chmod -R *": "deny",
"sudo *": "deny",
"doas *": "deny",
"npm install*": "deny",
"npm i": "deny",
"npm i *": "deny",
"npm ci*": "deny",
"pnpm install*": "deny",
"pnpm i": "deny",
"pnpm i *": "deny",
"pnpm add*": "deny",
"yarn add*": "deny",
"yarn install*": "deny",
"pip install*": "deny",
"pip3 install*": "deny",
"pipx install*": "deny",
"cargo install*": "deny",
"go install*": "deny",
"gem install*": "deny",
"brew install*": "deny",
"pacman *": "deny",
"yay *": "deny",
"apt *": "deny",
"apt-get *": "deny",
"dnf *": "deny",
"cat .env*": "deny",
"cat *.env": "deny",
"cat *.pem": "deny",
"cat *.key": "deny",
"cat *id_rsa*": "deny",
"cat *secret*": "deny",
"cat *credential*": "deny",
"cat ~/.ssh/*": "deny",
"cat ~/.aws/*": "deny",
"grep .env*": "deny",
"grep * .env*": "deny"
}
},
"provider": {
"lwcpp": {
"npm": "@ai-sdk/openai-compatible",
"name": "llamacpp (lw local)",
"options": {
"baseURL": "http://127.0.0.1:11343/v1"
},
"models": {
"gemma-4-E2B-it-UD-Q4_K_XL": { "name": "Gemma 4 E2B · 4k — fastest on lw (~6 t/s)", "limit": { "context": 4096, "output": 1024 } },
"Granite-4.0-H-Tiny": { "name": "Granite 4.0 H Tiny · 8k — speed AND brains (~4.8 t/s)", "limit": { "context": 8192, "output": 2048 } },
"Qwen3-1.7B": { "name": "Qwen3 1.7B · 8k — thinking, snappy (~4.8 t/s)", "limit": { "context": 8192, "output": 2048 } },
"Qwen3-4B-Instruct-2507": { "name": "Qwen3 4B 2507 · 4k — daily driver (~3.9 t/s)", "limit": { "context": 4096, "output": 1024 } },
"Qwen2.5-Coder-3B-Instruct": { "name": "Qwen2.5 Coder 3B · 8k — small coder (~3.3 t/s)", "limit": { "context": 8192, "output": 2048 } },
"Qwen3-30B-A3B-Instruct-2507-UD-IQ3_XXS": { "name": "Qwen3 30B A3B · 4k — quality when you can wait (~2.7 t/s)", "limit": { "context": 4096, "output": 1024 } },
"Jan-v3-4b": { "name": "Jan v3 4B · 4k — agentic tune (~2.6 t/s)", "limit": { "context": 4096, "output": 1024 } },
"gemma-4-E4B-it-UD-Q4_K_XL": { "name": "Gemma 4 E4B · 4k — quality; vision once mmproj added (~2.5 t/s)", "limit": { "context": 4096, "output": 1024 } }
}
},
"duskadiy": {
"npm": "@ai-sdk/openai-compatible",
"name": "DuskaDIY",
"options": {
"baseURL": "https://llm.duskadiy.com/api/v1"
},
"models": {
"Qwen3-Coder-30B-Instruct-UD-Q3_K_XL": { "name": "Qwen3 Coder 30B · 32k — main agent coder (remote)", "limit": { "context": 32768, "output": 4096 } },
"Qwen3-Coder-Next-UD-IQ3_XXS": { "name": "Qwen3 Coder Next 80B · 128k — long sessions (remote)", "limit": { "context": 131072, "output": 16384 } },
"Qwen3.6-35B-A3B-MTP-UD-IQ3_XXS": { "name": "Qwen3.6 35B · 24k · vision — daily driver (remote)", "attachment": true, "limit": { "context": 24576, "output": 8192 } },
"Qwen3.6-35B-A3B-Thinking": { "name": "Qwen3.6 35B Thinking · 24k · vision — hard problems (remote)", "attachment": true, "limit": { "context": 24576, "output": 8192 } },
"Qwen3.5-9B-UD-Q6_K_XL": { "name": "Qwen3.5 9B · 32k · vision — quick tasks (remote)", "attachment": true, "limit": { "context": 32768, "output": 8192 } },
"gemma-4-26B-A4B-it-UD-IQ4_XS": { "name": "Gemma 4 26B · 24k · vision — quality generalist (remote)", "attachment": true, "limit": { "context": 24576, "output": 4096 } },
"gemma-4-E4B-it-UD-Q8_K_XL": { "name": "Gemma 4 E4B · 64k · vision — fast generalist, long docs (remote)", "attachment": true, "limit": { "context": 65536, "output": 16384 } },
"GLM-4.7-Flash-UD-Q4_K_XL": { "name": "GLM-4.7 Flash · 24k — quality coder (remote)", "limit": { "context": 24576, "output": 8192 } },
"gpt-oss-20b": { "name": "gpt-oss 20B · 64k — fast reasoning + tools (remote)", "limit": { "context": 65536, "output": 8192 } },
"gpt-oss-20b-low": { "name": "gpt-oss 20B low · 64k — snappy answers (remote)", "limit": { "context": 65536, "output": 8192 } }
}
},
"llama.cpp": {
"npm": "@ai-sdk/openai-compatible",
"name": "llamacpp",
"options": {
"baseURL": "http://192.168.0.204:11343/v1"
},
"models": {
"Qwen3-Coder-Next-UD-IQ3_XXS": { "name": "Qwen3 Coder Next 80B · 128k — long sessions", "limit": { "context": 131072, "output": 16384 } },
"Qwen3-Coder-30B-Instruct-UD-Q3_K_XL": { "name": "Qwen3 Coder 30B · 32k — main agent coder", "limit": { "context": 32768, "output": 4096 } },
"Qwen3.6-35B-A3B-MTP-UD-IQ3_XXS": { "name": "Qwen3.6 35B · 24k · vision — daily driver", "attachment": true, "limit": { "context": 24576, "output": 8192 } },
"Qwen3.6-35B-A3B-Thinking": { "name": "Qwen3.6 35B Thinking · 24k · vision — hard problems", "attachment": true, "limit": { "context": 24576, "output": 8192 } },
"Qwen3.5-9B-UD-Q6_K_XL": { "name": "Qwen3.5 9B · 32k · vision — quick tasks", "attachment": true, "limit": { "context": 32768, "output": 8192 } },
"gemma-4-26B-A4B-it-UD-IQ4_XS": { "name": "Gemma 4 26B · 24k · vision — quality generalist", "attachment": true, "limit": { "context": 24576, "output": 4096 } },
"gemma-4-E4B-it-UD-Q8_K_XL": { "name": "Gemma 4 E4B · 64k · vision — fast generalist, long docs", "attachment": true, "limit": { "context": 65536, "output": 16384 } },
"GLM-4.7-Flash-UD-Q4_K_XL": { "name": "GLM-4.7 Flash · 24k — quality coder", "limit": { "context": 24576, "output": 8192 } },
"gpt-oss-20b": { "name": "gpt-oss 20B · 64k — fast reasoning + tools", "limit": { "context": 65536, "output": 8192 } },
"gpt-oss-20b-low": { "name": "gpt-oss 20B low · 64k — snappy answers", "limit": { "context": 65536, "output": 8192 } }
}
}
},
"agent": {
"plan": {
"mode": "primary",
"model": "llama.cpp/gemma-4-26B-A4B-it-UD-IQ4_XS",
"permission": {
"edit": "deny"
}
},
"build": {
"mode": "primary",
"model": "llama.cpp/gemma-4-26B-A4B-it-UD-IQ4_XS",
"permission": {
"edit": {
"*": "allow",
"**/.env*": "deny",
"**/*.env": "deny",
"**/*.pem": "deny",
"**/*.key": "deny",
"**/id_rsa*": "deny",
"**/.ssh/**": "deny",
"**/.aws/**": "deny",
"**/.git/**": "deny",
"**/.npmrc": "deny",
"**/.netrc": "deny"
}
}
},
"code-reviewer": {
"description": "Reviews code for best practices and potential issues",
"prompt": "You are a code reviewer. Focus on security, performance, and maintainability.",
"mode": "subagent",
"model": "llama.cpp/GLM-4.7-Flash-UD-Q4_K_XL",
"permission": {
"edit": "deny"
}
},
"test-runner": {
"description": "Runs and interprets test suites",
"prompt": "You are a test runner. Focus on identifying failing tests and providing clear reproduction steps.",
"mode": "subagent",
"model": "llama.cpp/GLM-4.7-Flash-UD-Q4_K_XL",
"permission": {
"edit": "deny"
}
},
"doc-writer": {
"description": "Maintains project documentation and docstrings",
"prompt": "You are a technical writer. Focus on clarity, accuracy, and keeping documentation in sync with code changes.",
"mode": "subagent",
"model": "llama.cpp/gemma-4-26B-A4B-it-UD-IQ4_XS",
"permission": {
"edit": {
"*": "deny",
"**/*.md": "allow",
"**/*.mdx": "allow",
"**/*.markdown": "allow",
"**/*.txt": "allow",
"**/*.rst": "allow"
}
}
},
"security-auditor": {
"description": "Scans for security vulnerabilities and hardcoded secrets",
"prompt": "You are a security auditor. Focus on OWASP principles, secret detection, and dependency vulnerabilities.",
"mode": "subagent",
"model": "llama.cpp/GLM-4.7-Flash-UD-Q4_K_XL",
"permission": {
"edit": "deny"
}
}
}
}
+40
View File
@@ -0,0 +1,40 @@
# F13 is a real keycode that no app binds. We declare it a virtual modifier so it can
# prefix the nav-layer chords below, then remap RightAlt -> F13 (see modmap). This mirrors
# the wm/fl evremap setup: RightAlt only ever emits F13, so a bare tap/hold never triggers
# app menu bars and no stray Alt leaks on unmapped chords.
virtual_modifiers:
- F13
modmap:
- name: CapsLock to LeftCtrl/Esc
remap:
CapsLock:
held: leftctrl
alone: esc
alone_timeout_millis: 150
- name: RightAlt to F13 (nav-layer modifier; Left Alt untouched)
remap:
Alt_R: F13
keymap:
- name: RightAlt(F13) vim-nav layer — mirrors wm/fl evremap
remap:
F13-h: left
F13-j: down
F13-k: up
F13-l: right
F13-u: pageup
F13-d: pagedown
F13-b: C-left # word left (Ctrl+Left)
F13-w: C-right # word right (Ctrl+Right)
# keymap:
# - name: Freetube remap
# application:
# only: FreeTube
# remap:
# Alt-h: Left
# Alt-l: Right
# Alt-j: Down
# Alt-k: Up
+299
View File
@@ -0,0 +1,299 @@
{
"providers": {
"lwcpp": {
"baseUrl": "http://127.0.0.1:11343/v1",
"api": "openai-completions",
"apiKey": "no-key-required",
"compat": {
"supportsDeveloperRole": false,
"supportsReasoningEffort": false,
"maxTokensField": "max_tokens"
},
"models": [
{
"id": "gemma-4-E2B-it-UD-Q4_K_XL",
"name": "Gemma 4 E2B · 4k — fastest on lw (~6 t/s)",
"reasoning": false,
"input": ["text"],
"contextWindow": 4096,
"maxTokens": 1024,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "Granite-4.0-H-Tiny",
"name": "Granite 4.0 H Tiny · 8k — speed AND brains (~4.8 t/s)",
"reasoning": false,
"input": ["text"],
"contextWindow": 8192,
"maxTokens": 2048,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "Qwen3-1.7B",
"name": "Qwen3 1.7B · 8k — thinking, snappy (~4.8 t/s)",
"reasoning": true,
"compat": { "thinkingFormat": "qwen-chat-template" },
"input": ["text"],
"contextWindow": 8192,
"maxTokens": 2048,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "Qwen3-4B-Instruct-2507",
"name": "Qwen3 4B 2507 · 4k — daily driver (~3.9 t/s)",
"reasoning": false,
"input": ["text"],
"contextWindow": 4096,
"maxTokens": 1024,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "Qwen2.5-Coder-3B-Instruct",
"name": "Qwen2.5 Coder 3B · 8k — small coder (~3.3 t/s)",
"reasoning": false,
"input": ["text"],
"contextWindow": 8192,
"maxTokens": 2048,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "Qwen3-30B-A3B-Instruct-2507-UD-IQ3_XXS",
"name": "Qwen3 30B A3B · 4k — quality when you can wait (~2.7 t/s)",
"reasoning": false,
"input": ["text"],
"contextWindow": 4096,
"maxTokens": 1024,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "Jan-v3-4b",
"name": "Jan v3 4B · 4k — agentic tune (~2.6 t/s)",
"reasoning": false,
"input": ["text"],
"contextWindow": 4096,
"maxTokens": 1024,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "gemma-4-E4B-it-UD-Q4_K_XL",
"name": "Gemma 4 E4B · 4k — quality; vision once mmproj added (~2.5 t/s)",
"reasoning": false,
"input": ["text"],
"contextWindow": 4096,
"maxTokens": 1024,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
}
]
},
"duskadiy": {
"baseUrl": "https://llm.duskadiy.com/api/v1",
"api": "openai-completions",
"apiKey": "$DUSKADIY_API_KEY",
"compat": {
"supportsDeveloperRole": false,
"supportsReasoningEffort": false,
"maxTokensField": "max_tokens"
},
"models": [
{
"id": "gemma-4-E4B-it-UD-Q8_K_XL",
"name": "Gemma 4 E4B · 64k · vision — fast generalist, long docs",
"reasoning": false,
"input": ["text", "image"],
"contextWindow": 65536,
"maxTokens": 16384,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "gemma-4-26B-A4B-it-UD-IQ4_XS",
"name": "Gemma 4 26B · 24k · vision — quality generalist",
"reasoning": false,
"input": ["text", "image"],
"contextWindow": 24576,
"maxTokens": 4096,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "Qwen3-Coder-30B-Instruct-UD-Q3_K_XL",
"name": "Qwen3 Coder 30B · 32k — main agent coder",
"reasoning": false,
"input": ["text"],
"contextWindow": 32768,
"maxTokens": 4096,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "Qwen3-Coder-Next-UD-IQ3_XXS",
"name": "Qwen3 Coder Next 80B · 128k — long sessions",
"reasoning": false,
"input": ["text"],
"contextWindow": 131072,
"maxTokens": 16384,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "Qwen3.6-35B-A3B-MTP-UD-IQ3_XXS",
"name": "Qwen3.6 35B · 24k · vision — daily driver",
"reasoning": false,
"input": ["text", "image"],
"contextWindow": 24576,
"maxTokens": 8192,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "Qwen3.6-35B-A3B-Thinking",
"name": "Qwen3.6 35B Thinking · 24k · vision — hard problems",
"reasoning": true,
"compat": { "thinkingFormat": "qwen-chat-template" },
"input": ["text", "image"],
"contextWindow": 24576,
"maxTokens": 8192,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "Qwen3.5-9B-UD-Q6_K_XL",
"name": "Qwen3.5 9B · 32k · vision — quick tasks",
"reasoning": true,
"compat": { "thinkingFormat": "qwen-chat-template" },
"input": ["text", "image"],
"contextWindow": 32768,
"maxTokens": 8192,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "GLM-4.7-Flash-UD-Q4_K_XL",
"name": "GLM-4.7 Flash · 24k — quality coder",
"reasoning": true,
"compat": { "thinkingFormat": "qwen-chat-template" },
"input": ["text"],
"contextWindow": 24576,
"maxTokens": 8192,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "gpt-oss-20b",
"name": "gpt-oss 20B · 64k — fast reasoning + tools",
"reasoning": true,
"input": ["text"],
"contextWindow": 65536,
"maxTokens": 8192,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "gpt-oss-20b-low",
"name": "gpt-oss 20B low · 64k — snappy answers",
"reasoning": true,
"input": ["text"],
"contextWindow": 65536,
"maxTokens": 8192,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
}
]
},
"localcpp": {
"baseUrl": "http://192.168.0.204:11343/v1",
"api": "openai-completions",
"apiKey": "no-key-required",
"compat": {
"supportsDeveloperRole": false,
"supportsReasoningEffort": false,
"maxTokensField": "max_tokens"
},
"models": [
{
"id": "gemma-4-E4B-it-UD-Q8_K_XL",
"name": "Gemma 4 E4B · 64k · vision — fast generalist, long docs",
"reasoning": false,
"input": ["text", "image"],
"contextWindow": 65536,
"maxTokens": 16384,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "gemma-4-26B-A4B-it-UD-IQ4_XS",
"name": "Gemma 4 26B · 24k · vision — quality generalist",
"reasoning": false,
"input": ["text", "image"],
"contextWindow": 24576,
"maxTokens": 4096,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "Qwen3-Coder-30B-Instruct-UD-Q3_K_XL",
"name": "Qwen3 Coder 30B · 32k — main agent coder",
"reasoning": false,
"input": ["text"],
"contextWindow": 32768,
"maxTokens": 4096,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "Qwen3-Coder-Next-UD-IQ3_XXS",
"name": "Qwen3 Coder Next 80B · 128k — long sessions",
"reasoning": false,
"input": ["text"],
"contextWindow": 131072,
"maxTokens": 16384,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "Qwen3.6-35B-A3B-MTP-UD-IQ3_XXS",
"name": "Qwen3.6 35B · 24k · vision — daily driver",
"reasoning": false,
"input": ["text", "image"],
"contextWindow": 24576,
"maxTokens": 8192,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "Qwen3.6-35B-A3B-Thinking",
"name": "Qwen3.6 35B Thinking · 24k · vision — hard problems",
"reasoning": true,
"compat": { "thinkingFormat": "qwen-chat-template" },
"input": ["text", "image"],
"contextWindow": 24576,
"maxTokens": 8192,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "Qwen3.5-9B-UD-Q6_K_XL",
"name": "Qwen3.5 9B · 32k · vision — quick tasks",
"reasoning": true,
"compat": { "thinkingFormat": "qwen-chat-template" },
"input": ["text", "image"],
"contextWindow": 32768,
"maxTokens": 8192,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "GLM-4.7-Flash-UD-Q4_K_XL",
"name": "GLM-4.7 Flash · 24k — quality coder",
"reasoning": true,
"compat": { "thinkingFormat": "qwen-chat-template" },
"input": ["text"],
"contextWindow": 24576,
"maxTokens": 8192,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "gpt-oss-20b",
"name": "gpt-oss 20B · 64k — fast reasoning + tools",
"reasoning": true,
"input": ["text"],
"contextWindow": 65536,
"maxTokens": 8192,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
},
{
"id": "gpt-oss-20b-low",
"name": "gpt-oss 20B low · 64k — snappy answers",
"reasoning": true,
"input": ["text"],
"contextWindow": 65536,
"maxTokens": 8192,
"cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 }
}
]
}
}
}
+37
View File
@@ -0,0 +1,37 @@
{
"defaultProvider": "duskadiy",
"defaultModel": "gemma-4-26B-A4B-it-UD-IQ4_XS",
"defaultThinkingLevel": "off",
"theme": "catppuccin-mocha",
"quietStartup": false,
"defaultProjectTrust": "ask",
"enableInstallTelemetry": false,
"enabledModels": [
"lwcpp/*",
"localcpp/*",
"duskadiy/*"
],
"compaction": {
"enabled": true,
"reserveTokens": 6144,
"keepRecentTokens": 6000
},
"retry": {
"enabled": true,
"maxRetries": 3,
"baseDelayMs": 2000,
"provider": {
"timeoutMs": 3600000
}
},
"httpIdleTimeoutMs": 600000,
"npmCommand": [
"fnm",
"exec",
"--using=22",
"--",
"npm"
],
"showHardwareCursor": true,
"lastChangelogVersion": "0.83.0"
}