[Sync] adopt unified stow layout from the private repo
Mirrors the private dots tree at 900bdda: one shared base plus per-host overlays, replacing the old flat .config/ layout (last synced 2026-06-28). - packages: common/ gui/ wm/ lw/ fl/ + install.sh and bin/ tooling (dotsync, reconcile-hyde.sh) - new README (layout, deploy order, HyDE dependency), plus ToDo.md and HYDE-UPDATE.md - current HyDE waybar rig (layouts/, cava), pi agent extensions, claude/ config, tmux, presenterm, aichat roles - drops stale duplicates and generated cruft that should never have been tracked: the second top-level .pi/ copy, btop.log, zellij config.kdl.bak, fish_variables, nvim codecompanion.lua - .pi/agent/auth.json is gitignored now; auth.json.example ships instead - fl/ and wm/ hypr themes/ stay untracked (HyDE-generated per machine, per the root .gitignore)
This commit is contained in:
coja
Executable
+320
@@ -0,0 +1,320 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
llama.cpp router benchmark → writes a results ledger for later tuning.
|
||||
|
||||
SAFETY-FIRST: this GPU also drives the display — VRAM overcommit spills to GTT and can
|
||||
freeze the whole PC. The script therefore:
|
||||
- records the BASELINE VRAM before touching anything (leftover model / browser skews everything)
|
||||
- ensures a CLEAN card before EVERY load (incl. the first): resident models are force-unloaded
|
||||
via the router API (POST /models/unload), then VRAM is verified with rocm-smi — the poll
|
||||
covers driver reclaim lag, older builds without the endpoint (sleep-idle fallback), and
|
||||
desktop apps the router doesn't own. Aborts if the card never drains.
|
||||
- checks free VRAM right after a model mounts; too tight (--min-free-gb) or GTT ballooning
|
||||
(--max-gtt-gb, measured against a PER-MODEL baseline — GTT reclaims slowly) → records a
|
||||
⚠ row and skips generating; the sweep continues after the drain-wait
|
||||
- refuses to sweep multiple models blind (no rocm-smi) unless --no-vram-ok
|
||||
|
||||
Per model it records decode/prefill t/s, TTFT, VRAM used/free, and the preset's knobs
|
||||
(ctx, n-cpu-moe, spec-type) parsed from config.ini.
|
||||
|
||||
Output: bench-results.md (latest run, overwritten) + bench-history.md (every run, appended)
|
||||
— both next to this script. Run ON the GPU box. Stdlib only.
|
||||
|
||||
./bench.py # all models the server lists
|
||||
./bench.py -m id1,id2 # only these
|
||||
./bench.py -x Qwen3-Coder-Next-UD-IQ3_XXS # skip some (e.g. not downloaded yet)
|
||||
./bench.py -n 512 --ctx 16000 # longer gen + a long-context decode column
|
||||
Compare a change: edit config.ini → reload server → re-run → diff runs in bench-history.md.
|
||||
"""
|
||||
import argparse, json, os, re, subprocess, sys, time, urllib.request, urllib.error
|
||||
from datetime import datetime
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
_smi_warned = False
|
||||
|
||||
|
||||
def rocm_mem(kind="vram"):
|
||||
"""(used_bytes, total_bytes) from rocm-smi for 'vram' or 'gtt'; (None, None) if unavailable."""
|
||||
global _smi_warned
|
||||
try:
|
||||
out = subprocess.run(["rocm-smi", "--showmeminfo", kind],
|
||||
capture_output=True, text=True, timeout=15).stdout
|
||||
except Exception:
|
||||
out = ""
|
||||
def grab(pat):
|
||||
m = re.search(pat + r"\s*:?\s*(\d+)", out)
|
||||
return int(m.group(1)) if m else None
|
||||
label = "VRAM" if kind == "vram" else "GTT"
|
||||
used = grab(label + r" Total Used Memory \(B\)") or grab(r"Used Memory \(B\)")
|
||||
total = grab(label + r" Total Memory \(B\)")
|
||||
if kind == "vram" and total is None and not _smi_warned:
|
||||
_smi_warned = True
|
||||
print(" (rocm-smi not readable here — VRAM columns and freeze guards are OFF)", file=sys.stderr)
|
||||
return used, total
|
||||
|
||||
|
||||
def with_retries(fn, retries, wait, label=""):
|
||||
"""Retry on 5xx/connection errors (model still mounting). A deterministic 'failed to
|
||||
load' from the router is NOT retried — that's a broken preset, not a slow mount."""
|
||||
last = None
|
||||
for i in range(retries + 1):
|
||||
try:
|
||||
return fn()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code < 500: # 4xx = real client error
|
||||
raise
|
||||
body = ""
|
||||
try:
|
||||
body = e.read().decode("utf-8", "ignore")
|
||||
except Exception:
|
||||
pass
|
||||
if "failed to load" in body: # deterministic → don't burn retries
|
||||
raise RuntimeError("model failed to load — check `journalctl -u llama.service`") from None
|
||||
last = e
|
||||
except (urllib.error.URLError, ConnectionError, TimeoutError) as e:
|
||||
last = e
|
||||
if i < retries:
|
||||
print(f" … {label} not ready ({last}); waiting {wait}s (try {i + 1}/{retries})", file=sys.stderr)
|
||||
time.sleep(wait)
|
||||
raise last
|
||||
|
||||
|
||||
def _open(url, key, payload=None, timeout=600):
|
||||
data = json.dumps(payload).encode() if payload is not None else None
|
||||
hdrs = {"Authorization": f"Bearer {key}"}
|
||||
if data:
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
return urllib.request.urlopen(urllib.request.Request(url, data=data, headers=hdrs), timeout=timeout)
|
||||
|
||||
|
||||
def list_models(base, key):
|
||||
d = json.load(_open(base.rstrip("/") + "/models", key))
|
||||
return [m["id"] for m in d.get("data", [])]
|
||||
|
||||
|
||||
def force_unload(base, key):
|
||||
"""Ask the router to unload every resident model (POST /models/unload on the ROOT api,
|
||||
not /v1). Harmless no-op on builds without the endpoint — callers fall back to polling."""
|
||||
root = re.sub(r"/v1/?$", "", base.rstrip("/"))
|
||||
try:
|
||||
d = json.load(_open(root + "/models", key, timeout=30))
|
||||
except Exception as e:
|
||||
print(f" (router /models not readable: {e} — relying on sleep-idle drain)", file=sys.stderr)
|
||||
return
|
||||
for m in d.get("data", []):
|
||||
if m.get("status", "loaded") != "loaded": # builds without 'status': try them all
|
||||
continue
|
||||
try:
|
||||
_open(root + "/models/unload", key, {"model": m["id"]}, timeout=30).read()
|
||||
except Exception:
|
||||
pass # not loaded / endpoint missing — poll covers it
|
||||
|
||||
|
||||
def parse_config(path):
|
||||
"""{preset: {ctx, ncpumoe, spec}} from the router config.ini."""
|
||||
sections, sec = {}, None
|
||||
try:
|
||||
lines = open(path, encoding="utf-8").read().splitlines()
|
||||
except OSError:
|
||||
return {}
|
||||
for ln in lines:
|
||||
s = ln.strip()
|
||||
if s.startswith("[") and s.endswith("]"):
|
||||
sec = s[1:-1]; sections[sec] = {}
|
||||
elif sec and "=" in s and not s.startswith("#"):
|
||||
k, v = s.split("=", 1)
|
||||
sections[sec][k.strip()] = v.split("#", 1)[0].strip()
|
||||
return {sec: {"ctx": kv.get("ctx-size", "-"),
|
||||
"ncpumoe": kv.get("n-cpu-moe", "-"),
|
||||
"spec": kv.get("spec-type", "-") or "-"}
|
||||
for sec, kv in sections.items()}
|
||||
|
||||
|
||||
def run(base, key, model, prompt, n, temp, timeout):
|
||||
payload = {"model": model, "messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": n, "temperature": temp, "stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
# llama.cpp extension: makes the server embed its own timings in the stream →
|
||||
# the table shows the SAME predicted_per_second as the llama.cpp web UI
|
||||
# (without it we fall back to wall-clock estimates, which read lower).
|
||||
"timings_per_token": True}
|
||||
t0 = time.perf_counter(); ttft = None; ntok = 0; usage = None; timings = None
|
||||
for raw in _open(base.rstrip("/") + "/chat/completions", key, payload, timeout):
|
||||
line = raw.decode("utf-8", "ignore").strip()
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
body = line[5:].strip()
|
||||
if body == "[DONE]":
|
||||
break
|
||||
try:
|
||||
c = json.loads(body)
|
||||
except ValueError:
|
||||
continue
|
||||
ch = c.get("choices") or [{}]
|
||||
if ch and ch[0].get("delta", {}).get("content"):
|
||||
if ttft is None:
|
||||
ttft = time.perf_counter() - t0
|
||||
ntok += 1
|
||||
if c.get("usage"):
|
||||
usage = c["usage"]
|
||||
if c.get("timings"):
|
||||
timings = c["timings"]
|
||||
total = time.perf_counter() - t0
|
||||
comp = (usage or {}).get("completion_tokens") or ntok
|
||||
ptok = (usage or {}).get("prompt_tokens")
|
||||
gen_s = total - (ttft or total)
|
||||
tg = comp / gen_s if gen_s > 0 else 0.0
|
||||
pp = (ptok / ttft) if (ptok and ttft) else None
|
||||
if timings: # llama.cpp's own numbers are authoritative when the router forwards them
|
||||
tg = timings.get("predicted_per_second", tg)
|
||||
pp = timings.get("prompt_per_second", pp)
|
||||
ptok = timings.get("prompt_n", ptok)
|
||||
return {"tg": tg, "pp": pp, "ttft": ttft or 0.0, "ptok": ptok or 0}
|
||||
|
||||
|
||||
def wait_drain(drain_gb, window_s, label="", base=None, key=None):
|
||||
"""Force-unload resident models via the router API, then poll until VRAM used drops
|
||||
below drain_gb (card is clean) or the window expires."""
|
||||
used, _ = rocm_mem()
|
||||
if base and used is not None and used > drain_gb * 1e9:
|
||||
force_unload(base, key)
|
||||
end = time.time() + window_s
|
||||
announced = False
|
||||
while used is not None and used > drain_gb * 1e9 and time.time() < end:
|
||||
if not announced:
|
||||
print(f" … waiting for VRAM to drain before {label} ({gb(used)} used)", file=sys.stderr)
|
||||
announced = True
|
||||
time.sleep(5)
|
||||
used, _ = rocm_mem()
|
||||
return used
|
||||
|
||||
|
||||
def gb(b):
|
||||
return f"{b / 1e9:.1f} GB" if b else "-"
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("-u", "--url", default="http://192.168.0.204:11343/v1")
|
||||
ap.add_argument("-k", "--key", default="no-key-required")
|
||||
ap.add_argument("-m", "--models", default="all", help="comma-separated ids, or 'all'")
|
||||
ap.add_argument("-x", "--skip", default="", help="comma-separated ids to skip")
|
||||
ap.add_argument("-n", "--tokens", type=int, default=256)
|
||||
ap.add_argument("-t", "--temp", type=float, default=0.3)
|
||||
ap.add_argument("--ctx", type=int, default=0, help="also record decode at ~this many prompt tokens")
|
||||
ap.add_argument("--timeout", type=int, default=600, help="per-request stall timeout (s)")
|
||||
ap.add_argument("--config", default=os.path.join(HERE, "config.ini"))
|
||||
ap.add_argument("--out", default=os.path.join(HERE, "bench-results.md"))
|
||||
ap.add_argument("--history", default=os.path.join(HERE, "bench-history.md"))
|
||||
ap.add_argument("--retries", type=int, default=4, help="retries while a model is still mounting")
|
||||
ap.add_argument("--retry-wait", type=int, default=15, help="seconds between retries")
|
||||
ap.add_argument("--drain-gb", type=float, default=4.0,
|
||||
help="VRAM considered 'clean' below this (raise it if your desktop alone uses more)")
|
||||
ap.add_argument("--min-free-gb", type=float, default=1.5,
|
||||
help="SAFETY: stop the sweep if a loaded model leaves less VRAM free than this")
|
||||
ap.add_argument("--max-gtt-gb", type=float, default=2.0,
|
||||
help="SAFETY: flag a model if GTT grows more than this over baseline (= VRAM spilling to RAM)")
|
||||
ap.add_argument("--unload-wait", type=int, default=360,
|
||||
help="after a too-tight model, wait up to this long for it to unload before continuing "
|
||||
"(sleep-idle-seconds keeps models resident up to 300s)")
|
||||
ap.add_argument("--no-vram-ok", action="store_true",
|
||||
help="allow a multi-model sweep even though rocm-smi (and thus the freeze guards) is unavailable")
|
||||
a = ap.parse_args()
|
||||
|
||||
models = list_models(a.url, a.key) if a.models == "all" else [m.strip() for m in a.models.split(",")]
|
||||
skip = {s.strip() for s in a.skip.split(",") if s.strip()}
|
||||
models = [m for m in models if m not in skip]
|
||||
cfg = parse_config(a.config)
|
||||
|
||||
used0, total_vram = rocm_mem("vram")
|
||||
if total_vram is None and len(models) > 1 and not a.no_vram_ok:
|
||||
sys.exit("ABORT: rocm-smi is not readable here, so the VRAM/GTT freeze guards can't work.\n"
|
||||
"Run this ON the GPU box, bench a single model with -m <id>, or pass --no-vram-ok to accept the risk.")
|
||||
if used0 is not None and used0 > a.drain_gb * 1e9:
|
||||
print(f"⚠ baseline VRAM used is already {gb(used0)} — a leftover model (sleep-idle keeps them "
|
||||
f"up to 5 min) or desktop apps (browser!). Results/safety margins will be skewed; "
|
||||
f"ideally close GPU apps or wait, then re-run.", file=sys.stderr)
|
||||
|
||||
task = "Write a Python function that merges two sorted lists, with a short docstring and one example."
|
||||
filler = ("The quick brown fox jumps over the lazy dog. " * max(1, a.ctx // 9)) if a.ctx else None
|
||||
|
||||
cols = ["model", "cfg ctx", "n-cpu-moe", "spec", "decode t/s", "prefill t/s", "TTFT s", "VRAM used", "VRAM free"]
|
||||
if a.ctx:
|
||||
cols.append(f"decode@{a.ctx // 1000}k")
|
||||
|
||||
latest = open(a.out, "w", encoding="utf-8")
|
||||
hist = open(a.history, "a", encoding="utf-8")
|
||||
meta = (f"_Run {datetime.now():%Y-%m-%d %H:%M} · server `{a.url}` · VRAM total {gb(total_vram)} · "
|
||||
f"baseline used {gb(used0)} · gen {a.tokens} tok · guards: free≥{a.min_free_gb} GB, GTT+≤{a.max_gtt_gb} GB_")
|
||||
|
||||
def emit(line, both=True):
|
||||
print(line)
|
||||
latest.write(line + "\n"); latest.flush()
|
||||
if both:
|
||||
hist.write(line + "\n"); hist.flush()
|
||||
|
||||
latest.write("# llama.cpp benchmark results (latest run)\n\n")
|
||||
hist.write(f"\n## run {datetime.now():%Y-%m-%d %H:%M}\n\n")
|
||||
emit(meta + "\n")
|
||||
emit("| " + " | ".join(cols) + " |")
|
||||
emit("|" + "|".join(["---"] * len(cols)) + "|")
|
||||
|
||||
for m in models:
|
||||
c = cfg.get(m, {})
|
||||
base_cells = [m, c.get("ctx", "-"), c.get("ncpumoe", "-"), c.get("spec", "-")]
|
||||
# A CLEAN card before EVERY load (incl. the first): force-unload via the router API,
|
||||
# then verify with rocm-smi — double residency = bogus numbers or a freeze.
|
||||
u = wait_drain(a.drain_gb, a.unload_wait, m, a.url, a.key)
|
||||
if u is not None and u > a.drain_gb * 1e9:
|
||||
emit("")
|
||||
emit(f"_⚠ aborted before {m}: VRAM still {gb(u)} used after {a.unload_wait}s — something "
|
||||
f"won't unload (leftover model / GPU apps). If that's your normal desktop, raise --drain-gb._")
|
||||
print(f"\n⚠ aborted: card not clean ({gb(u)} used).", file=sys.stderr)
|
||||
break
|
||||
gtt_pre, _ = rocm_mem("gtt") # per-model GTT baseline — GTT reclaims slowly across swaps
|
||||
try:
|
||||
# warmup — retries wait out the mount; 'failed to load' aborts immediately
|
||||
with_retries(lambda: run(a.url, a.key, m, "hi", 8, a.temp, a.timeout), a.retries, a.retry_wait, m)
|
||||
used, _ = rocm_mem("vram")
|
||||
gtt, _ = rocm_mem("gtt")
|
||||
free = (total_vram - used) if (total_vram and used) else None
|
||||
gtt_delta = (gtt - gtt_pre) if (gtt is not None and gtt_pre is not None) else None
|
||||
reason = None
|
||||
if free is not None and free < a.min_free_gb * 1e9:
|
||||
reason = f"only {gb(free)} VRAM free"
|
||||
elif gtt_delta is not None and gtt_delta > a.max_gtt_gb * 1e9:
|
||||
reason = f"GTT grew {gb(gtt_delta)} = spilling to system RAM"
|
||||
if reason:
|
||||
# Too tight — do NOT generate (allocates more). Record the row; the sweep continues
|
||||
# once this model has fully unloaded (see below).
|
||||
hint = ("raise n-cpu-moe" if c.get("ncpumoe", "-") != "-"
|
||||
else "lower ctx / smaller quant / add n-cpu-moe if MoE")
|
||||
cells = base_cells + [f"⚠ {reason} — SKIPPED ({hint})", "", "", gb(used), gb(free)]
|
||||
if a.ctx:
|
||||
cells.append("")
|
||||
else:
|
||||
r = with_retries(lambda: run(a.url, a.key, m, task, a.tokens, a.temp, a.timeout),
|
||||
a.retries, a.retry_wait, m)
|
||||
cells = base_cells + [f"{r['tg']:.1f}", (f"{r['pp']:.0f}" if r['pp'] else "-"),
|
||||
f"{r['ttft']:.2f}", gb(used), gb(free)]
|
||||
if a.ctx:
|
||||
rc = with_retries(lambda: run(a.url, a.key, m, filler + "\n\n" + task, a.tokens, a.temp, a.timeout),
|
||||
a.retries, a.retry_wait, m)
|
||||
cells.append(f"{rc['tg']:.1f}")
|
||||
except Exception as e:
|
||||
err = f"HTTP {e.code} (check server log)" if isinstance(e, urllib.error.HTTPError) else f"ERROR {e}"
|
||||
cells = base_cells + [err] + [""] * (len(cols) - len(base_cells) - 1)
|
||||
emit("| " + " | ".join(str(x) for x in cells) + " |")
|
||||
# (no post-row wait needed: the pre-load drain above protects the next iteration,
|
||||
# including after ⚠-tight rows where generation was skipped)
|
||||
|
||||
latest.write("\n_Tuning hints: **VRAM free** = headroom to raise `ctx-size` or lower `n-cpu-moe` "
|
||||
"(more experts on GPU → faster). Low decode t/s on an offloaded MoE → lower `n-cpu-moe` "
|
||||
"if free allows. High **baseline used** invalidates the run. For spec/MTP models, check "
|
||||
"the server log's acceptance rate._\n")
|
||||
latest.close(); hist.close()
|
||||
print(f"\nwrote {a.out} (+ appended {a.history})")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user