coja
1e39e0cb4b
Supersedes the old flat .config/ layout (last published 2026-06-28) with the private repo's structure: one shared base plus per-host overlays. - packages: common/ gui/ lw/ fl/ wm/ plus install.sh and bin/ tooling (dotsync, reconcile-hyde.sh) - new README covering the layout, deploy order and the HyDE dependency - current HyDE waybar rig (layouts/, cava), pi agent extensions, claude/ config, tmux, presenterm, aichat roles - fish: kp (keepassxc-cli + fzf picker, db path from $KP_DB) and bind_M_n_history (alt+1..9 recalls the nth history entry) - drops cruft that should never have been tracked: the duplicate top-level .pi/ copy, btop.log, zellij config.kdl.bak, fish_variables - .pi/agent/auth.json is gitignored; auth.json.example ships instead Host-specific work sessions and the personal backlog stay in the private tree. Endpoint locators in the llamacpp/whisper guides are placeholders ($SERVER, <own-domain>) — the guides themselves stay, since they are the useful part.
263 lines
12 KiB
Python
Executable File
263 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""
|
||
llama.cpp router benchmark — LAPTOP (lw) edition → writes a results ledger for tuning.
|
||
|
||
This is the CPU analog of the fl (RX 7600 XT) bench.py. lw has NO discrete GPU, so all
|
||
the ROCm / VRAM / GTT / freeze-guard / force-unload machinery from the fl version is GONE
|
||
— on a CPU box there is no display-driving GPU to starve. The only resource ceiling is
|
||
system RAM, so instead of VRAM this records per-model **RAM footprint** (MemAvailable
|
||
delta, from /proc/meminfo) alongside decode/prefill/TTFT.
|
||
|
||
Per model it records decode/prefill t/s, TTFT, RAM used (Δ) / RAM free, and the preset's
|
||
knobs (ctx, threads, spec-type) parsed from config.ini. For spec/MTP presets the draft
|
||
acceptance % is appended to the spec column, read from the response's `timings` object
|
||
(the child server also prints a `draft acceptance` log line; the table is easier to compare).
|
||
|
||
Output: bench-results.md (latest run, overwritten) + bench-history.md (every run, appended)
|
||
— both next to this script. Run ON THE LAPTOP against its own server. Stdlib only.
|
||
|
||
./bench.py # all models the server lists
|
||
./bench.py -m id1,id2 # only these
|
||
./bench.py -x some-id # skip some (e.g. not downloaded yet)
|
||
./bench.py -n 512 --ctx 8000 # longer gen + a long-context decode column
|
||
|
||
Note: the laptop runs `--models-max 1` (LRU eviction), so mid-sweep the RAM Δ is NET —
|
||
each load evicts the previous model, and a smaller model after a bigger one reads negative.
|
||
Only the first model after a server restart shows a true footprint.
|
||
Compare a change: edit config.ini → restart server → re-run → diff runs in bench-history.md.
|
||
"""
|
||
import argparse, json, os, re, subprocess, sys, time, urllib.request, urllib.error
|
||
from datetime import datetime
|
||
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
|
||
|
||
def mem_available():
|
||
"""(available_bytes, total_bytes) from /proc/meminfo; (None, None) if unavailable."""
|
||
try:
|
||
info = {}
|
||
for ln in open("/proc/meminfo", encoding="utf-8"):
|
||
k, _, rest = ln.partition(":")
|
||
m = re.search(r"(\d+)\s*kB", rest)
|
||
if m:
|
||
info[k.strip()] = int(m.group(1)) * 1024
|
||
return info.get("MemAvailable"), info.get("MemTotal")
|
||
except OSError:
|
||
return None, None
|
||
|
||
|
||
def with_retries(fn, retries, wait, label=""):
|
||
"""Retry on 5xx/connection errors (model still mounting). A deterministic 'failed to
|
||
load' from the router is NOT retried — that's a broken preset, not a slow mount."""
|
||
last = None
|
||
for i in range(retries + 1):
|
||
try:
|
||
return fn()
|
||
except urllib.error.HTTPError as e:
|
||
if e.code < 500: # 4xx = real client error
|
||
raise
|
||
body = ""
|
||
try:
|
||
body = e.read().decode("utf-8", "ignore")
|
||
except Exception:
|
||
pass
|
||
if "failed to load" in body: # deterministic → don't burn retries
|
||
raise RuntimeError("model failed to load — check the llama-server log") from None
|
||
last = e
|
||
except (urllib.error.URLError, ConnectionError, TimeoutError) as e:
|
||
last = e
|
||
if i < retries:
|
||
print(f" … {label} not ready ({last}); waiting {wait}s (try {i + 1}/{retries})", file=sys.stderr)
|
||
time.sleep(wait)
|
||
raise last
|
||
|
||
|
||
def _open(url, key, payload=None, timeout=600):
|
||
data = json.dumps(payload).encode() if payload is not None else None
|
||
hdrs = {"Authorization": f"Bearer {key}"}
|
||
if data:
|
||
hdrs["Content-Type"] = "application/json"
|
||
return urllib.request.urlopen(urllib.request.Request(url, data=data, headers=hdrs), timeout=timeout)
|
||
|
||
|
||
def list_models(base, key):
|
||
d = json.load(_open(base.rstrip("/") + "/models", key))
|
||
return [m["id"] for m in d.get("data", [])]
|
||
|
||
|
||
def parse_config(path):
|
||
"""{preset: {ctx, threads, spec}} from the router config.ini."""
|
||
sections, sec = {}, None
|
||
try:
|
||
lines = open(path, encoding="utf-8").read().splitlines()
|
||
except OSError:
|
||
return {}
|
||
for ln in lines:
|
||
s = ln.strip()
|
||
if s.startswith("[") and s.endswith("]"):
|
||
sec = s[1:-1]; sections[sec] = {}
|
||
elif sec and "=" in s and not s.startswith("#"):
|
||
k, v = s.split("=", 1)
|
||
sections[sec][k.strip()] = v.split("#", 1)[0].strip()
|
||
return {sec: {"ctx": kv.get("ctx-size", "-"),
|
||
"threads": kv.get("threads", "-"),
|
||
"spec": kv.get("spec-type", "-") or "-"}
|
||
for sec, kv in sections.items()}
|
||
|
||
|
||
def run(base, key, model, prompt, n, temp, timeout):
|
||
payload = {"model": model, "messages": [{"role": "user", "content": prompt}],
|
||
"max_tokens": n, "temperature": temp, "stream": True,
|
||
"stream_options": {"include_usage": True},
|
||
# llama.cpp extension: makes the server embed its own timings in the stream →
|
||
# the table shows the SAME predicted_per_second as the llama.cpp web UI
|
||
# (without it we fall back to wall-clock estimates, which read lower).
|
||
"timings_per_token": True}
|
||
t0 = time.perf_counter(); ttft = None; ntok = 0; usage = None; timings = None
|
||
for raw in _open(base.rstrip("/") + "/chat/completions", key, payload, timeout):
|
||
line = raw.decode("utf-8", "ignore").strip()
|
||
if not line.startswith("data:"):
|
||
continue
|
||
body = line[5:].strip()
|
||
if body == "[DONE]":
|
||
break
|
||
try:
|
||
c = json.loads(body)
|
||
except ValueError:
|
||
continue
|
||
ch = c.get("choices") or [{}]
|
||
d = ch[0].get("delta", {}) if ch else {}
|
||
# reasoning models stream `reasoning_content` first and may never emit `content`
|
||
if d.get("content") or d.get("reasoning_content"):
|
||
if ttft is None:
|
||
ttft = time.perf_counter() - t0
|
||
ntok += 1
|
||
if c.get("usage"):
|
||
usage = c["usage"]
|
||
if c.get("timings"):
|
||
timings = c["timings"]
|
||
total = time.perf_counter() - t0
|
||
comp = (usage or {}).get("completion_tokens") or ntok
|
||
ptok = (usage or {}).get("prompt_tokens")
|
||
gen_s = total - (ttft or total)
|
||
tg = comp / gen_s if gen_s > 0 else 0.0
|
||
pp = (ptok / ttft) if (ptok and ttft) else None
|
||
acc = None
|
||
if timings: # llama.cpp's own numbers are authoritative when the router forwards them
|
||
tg = timings.get("predicted_per_second", tg)
|
||
pp = timings.get("prompt_per_second", pp)
|
||
ptok = timings.get("prompt_n", ptok)
|
||
if timings.get("prompt_ms"):
|
||
ttft = timings["prompt_ms"] / 1000
|
||
if timings.get("draft_n"): # spec decode active — mirror the child's acceptance stat
|
||
acc = timings.get("draft_n_accepted", 0) / timings["draft_n"]
|
||
return {"tg": tg, "pp": pp, "ttft": ttft or 0.0, "ptok": ptok or 0, "acc": acc}
|
||
|
||
|
||
def gb(b):
|
||
return f"{b / 1e9:.1f} GB" if b else "-"
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("-u", "--url", default="http://127.0.0.1:11343/v1",
|
||
help="run this ON the laptop against its own server (localhost)")
|
||
ap.add_argument("-k", "--key", default="no-key-required")
|
||
ap.add_argument("-m", "--models", default="all", help="comma-separated ids, or 'all'")
|
||
ap.add_argument("-x", "--skip", default="", help="comma-separated ids to skip")
|
||
ap.add_argument("-n", "--tokens", type=int, default=256)
|
||
ap.add_argument("-t", "--temp", type=float, default=0.3)
|
||
ap.add_argument("--ctx", type=int, default=0, help="also record decode at ~this many prompt tokens")
|
||
ap.add_argument("--timeout", type=int, default=600, help="per-request stall timeout (s)")
|
||
ap.add_argument("--config", default=os.path.join(HERE, "config.ini"))
|
||
ap.add_argument("--out", default=os.path.join(HERE, "bench-results.md"))
|
||
ap.add_argument("--history", default=os.path.join(HERE, "bench-history.md"))
|
||
ap.add_argument("--retries", type=int, default=4, help="retries while a model is still mounting")
|
||
ap.add_argument("--retry-wait", type=int, default=15, help="seconds between retries")
|
||
ap.add_argument("--settle", type=float, default=1.0,
|
||
help="seconds to wait after warmup before reading RAM (let allocation settle)")
|
||
a = ap.parse_args()
|
||
|
||
models = list_models(a.url, a.key) if a.models == "all" else [m.strip() for m in a.models.split(",")]
|
||
skip = {s.strip() for s in a.skip.split(",") if s.strip()}
|
||
models = [m for m in models if m not in skip]
|
||
cfg = parse_config(a.config)
|
||
|
||
avail0, total_ram = mem_available()
|
||
|
||
task = "Write a Python function that merges two sorted lists, with a short docstring and one example."
|
||
filler = ("The quick brown fox jumps over the lazy dog. " * max(1, a.ctx // 9)) if a.ctx else None
|
||
|
||
cols = ["model", "cfg ctx", "threads", "spec", "decode t/s", "prefill t/s", "TTFT s", "RAM Δ", "RAM free"]
|
||
if a.ctx:
|
||
cols.append(f"decode@{a.ctx // 1000}k")
|
||
|
||
latest = open(a.out, "w", encoding="utf-8")
|
||
hist = open(a.history, "a", encoding="utf-8")
|
||
meta = (f"_Run {datetime.now():%Y-%m-%d %H:%M} · server `{a.url}` · RAM total {gb(total_ram)} · "
|
||
f"baseline avail {gb(avail0)} · gen {a.tokens} tok (CPU inference, no GPU)_")
|
||
|
||
def emit(line, both=True):
|
||
print(line)
|
||
latest.write(line + "\n"); latest.flush()
|
||
if both:
|
||
hist.write(line + "\n"); hist.flush()
|
||
|
||
latest.write("# llama.cpp benchmark results — laptop (lw), latest run\n\n")
|
||
hist.write(f"\n## run {datetime.now():%Y-%m-%d %H:%M}\n\n")
|
||
emit(meta + "\n")
|
||
emit("| " + " | ".join(cols) + " |")
|
||
emit("|" + "|".join(["---"] * len(cols)) + "|")
|
||
|
||
for m in models:
|
||
c = cfg.get(m, {})
|
||
base_cells = [m, c.get("ctx", "-"), c.get("threads", "-"), c.get("spec", "-")]
|
||
pre_avail, _ = mem_available() # per-model baseline (a model may already be resident)
|
||
try:
|
||
# warmup — retries wait out the mount; 'failed to load' aborts immediately
|
||
with_retries(lambda: run(a.url, a.key, m, "hi", 8, a.temp, a.timeout), a.retries, a.retry_wait, m)
|
||
if a.settle:
|
||
time.sleep(a.settle)
|
||
post_avail, _ = mem_available()
|
||
free = post_avail
|
||
delta = (pre_avail - post_avail) if (pre_avail is not None and post_avail is not None) else None
|
||
# with --models-max 1 the Δ is net-of-eviction: negative = replaced a bigger model
|
||
if delta is None:
|
||
delta_cell = "-"
|
||
elif delta > 0.2e9:
|
||
delta_cell = gb(delta)
|
||
elif delta < -0.2e9:
|
||
delta_cell = f"-{gb(-delta)} (net swap)"
|
||
else:
|
||
delta_cell = "~0 (resident/net)"
|
||
r = with_retries(lambda: run(a.url, a.key, m, task, a.tokens, a.temp, a.timeout),
|
||
a.retries, a.retry_wait, m)
|
||
cells = base_cells.copy()
|
||
if r.get("acc") is not None:
|
||
cells[3] = f"{cells[3]} ({r['acc'] * 100:.0f}% acc)"
|
||
cells += [f"{r['tg']:.1f}", (f"{r['pp']:.0f}" if r['pp'] else "-"),
|
||
f"{r['ttft']:.2f}", delta_cell, gb(free)]
|
||
if a.ctx:
|
||
rc = with_retries(lambda: run(a.url, a.key, m, filler + "\n\n" + task, a.tokens, a.temp, a.timeout),
|
||
a.retries, a.retry_wait, m)
|
||
cells.append(f"{rc['tg']:.1f}")
|
||
except Exception as e:
|
||
err = f"HTTP {e.code} (check server log)" if isinstance(e, urllib.error.HTTPError) else f"ERROR {e}"
|
||
cells = base_cells + [err] + [""] * (len(cols) - len(base_cells) - 1)
|
||
emit("| " + " | ".join(str(x) for x in cells) + " |")
|
||
|
||
latest.write("\n_Tuning hints (CPU box): decode reads the model's active bytes every token — smaller "
|
||
"quants and small-active MoE are directly faster. **RAM free** is the real ceiling. "
|
||
"Sanity-check: decode t/s × model GB = effective GB/s; dense models measure ~8-10 GB/s "
|
||
"here (the practical wall), a model far below that is compute-bound (MoE/hybrid) → more "
|
||
"`threads` may help it; `threads-batch = 4` may help prefill either way. With "
|
||
"`--models-max 1`, mid-sweep RAM Δ is net-of-eviction. For spec/MTP models the draft "
|
||
"acceptance % is shown in the spec column. ⚠ Absolute numbers swing ±20-30% with desktop "
|
||
"load/thermals — only same-run rows are directly comparable._\n")
|
||
latest.close(); hist.close()
|
||
print(f"\nwrote {a.out} (+ appended {a.history})")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|