[Sync] Lua Hyprland chain, licence cleanup

wm/.config/llamacpp/bench.py

@@ -0,0 +1,348 @@
+#!/usr/bin/env python3
+"""
+llama.cpp router benchmark — WM (GTX 1080 Ti) edition → writes a results ledger for tuning.
+
+This is the NVIDIA port of the fl (RX 7600 XT) bench.py. Same flow, different failure mode:
+an NVIDIA overcommit does NOT spill to system RAM and freeze the desktop — cudaMalloc fails
+and the load errors out cleanly. So the guards here protect the NUMBERS, not the PC:
+  - records the BASELINE VRAM after unloading anything of ours (the desktop alone holds 3.5–4.5 GB;
+    a browser/game on top skews every margin — the header says when a resident model was unloaded
+    first, so "7.5 GB baseline" can't masquerade as the desktop again)
+  - ensures a CLEAN card before EVERY load: resident models are force-unloaded via the
+    router API (POST /models/unload), then VRAM is verified with nvidia-smi — the poll covers
+    driver reclaim lag, older builds without the endpoint (sleep-idle fallback) and desktop
+    apps the router doesn't own. Aborts if the card never drains.
+  - checks free VRAM right after a model mounts; too tight (--min-free-gb) → records a ⚠ row
+    and skips generating (a preset that barely loads on a clean bench will fail on a busy
+    desktop); the sweep continues after the drain-wait
+  - refuses to sweep multiple models blind (no nvidia-smi) unless --no-vram-ok
+(There is no GTT column: that is an amdgpu concept. The gtt hooks are kept as no-ops so the
+ fl script and this one stay diffable.)
+
+Per model it records decode/prefill t/s, TTFT, VRAM used/free, and the preset's knobs
+(ctx, n-cpu-moe, spec-type) parsed from config.ini. For spec/MTP presets the draft
+acceptance % is appended to the spec column, read from the response's `timings` object.
+
+Output: bench-results.md (latest run, overwritten) + bench-history.md (every run, appended)
+— both next to this script. Run ON this box against its own localhost router. Stdlib only.
+
+  ./bench.py                            # all models the server lists
+  ./bench.py -m id1,id2                 # only these
+  ./bench.py -x some-id                 # skip some (e.g. not downloaded yet)
+  ./bench.py -n 512 --ctx 12000         # longer gen + a long-context decode column
+Compare a change: edit config.ini → restart `llamaserver` → re-run → diff runs in bench-history.md.
+"""
+import argparse, json, os, re, subprocess, sys, time, urllib.request, urllib.error
+from datetime import datetime
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+_smi_warned = False
+
+
+def gpu_mem(kind="vram"):
+    """(used_bytes, total_bytes) from nvidia-smi for 'vram'; (None, None) for 'gtt' (no such
+    thing on NVIDIA — kept so the fl flow stays identical) or if nvidia-smi is unavailable."""
+    global _smi_warned
+    if kind != "vram":
+        return None, None
+    try:
+        out = subprocess.run(["nvidia-smi", "--query-gpu=memory.used,memory.total",
+                              "--format=csv,noheader,nounits"],
+                             capture_output=True, text=True, timeout=15).stdout
+        used_mib, total_mib = (int(x.strip()) for x in out.strip().splitlines()[0].split(","))
+        return used_mib * 1024 * 1024, total_mib * 1024 * 1024
+    except Exception:
+        if not _smi_warned:
+            _smi_warned = True
+            print("  (nvidia-smi not readable here — VRAM columns and guards are OFF)", file=sys.stderr)
+        return None, None
+
+
+def with_retries(fn, retries, wait, label=""):
+    """Retry on 5xx/connection errors (model still mounting). A deterministic 'failed to
+    load' from the router is NOT retried — that's a broken preset, not a slow mount."""
+    last = None
+    for i in range(retries + 1):
+        try:
+            return fn()
+        except urllib.error.HTTPError as e:
+            if e.code < 500:              # 4xx = real client error
+                raise
+            body = ""
+            try:
+                body = e.read().decode("utf-8", "ignore")
+            except Exception:
+                pass
+            if "failed to load" in body:  # deterministic → don't burn retries
+                raise RuntimeError("model failed to load — check the llamaserver terminal (VRAM? path?)") from None
+            last = e
+        except (urllib.error.URLError, ConnectionError, TimeoutError) as e:
+            last = e
+        if i < retries:
+            print(f"    … {label} not ready ({last}); waiting {wait}s (try {i + 1}/{retries})", file=sys.stderr)
+            time.sleep(wait)
+    raise last
+
+
+def _open(url, key, payload=None, timeout=600):
+    data = json.dumps(payload).encode() if payload is not None else None
+    hdrs = {"Authorization": f"Bearer {key}"}
+    if data:
+        hdrs["Content-Type"] = "application/json"
+    return urllib.request.urlopen(urllib.request.Request(url, data=data, headers=hdrs), timeout=timeout)
+
+
+def list_models(base, key):
+    d = json.load(_open(base.rstrip("/") + "/models", key))
+    return [m["id"] for m in d.get("data", [])]
+
+
+def force_unload(base, key):
+    """Ask the router to unload every resident model (POST /models/unload on the ROOT api,
+    not /v1). Harmless no-op on builds without the endpoint — callers fall back to polling."""
+    root = re.sub(r"/v1/?$", "", base.rstrip("/"))
+    try:
+        d = json.load(_open(root + "/models", key, timeout=30))
+    except Exception as e:
+        print(f"    (router /models not readable: {e} — relying on sleep-idle drain)", file=sys.stderr)
+        return
+    for m in d.get("data", []):
+        if m.get("status", "loaded") != "loaded":   # builds without 'status': try them all
+            continue
+        try:
+            _open(root + "/models/unload", key, {"model": m["id"]}, timeout=30).read()
+        except Exception:
+            pass                                    # not loaded / endpoint missing — poll covers it
+
+
+def parse_config(path):
+    """{preset: {ctx, ncpumoe, spec}} from the router config.ini."""
+    sections, sec = {}, None
+    try:
+        lines = open(path, encoding="utf-8").read().splitlines()
+    except OSError:
+        return {}
+    for ln in lines:
+        s = ln.strip()
+        if s.startswith("[") and s.endswith("]"):
+            sec = s[1:-1]; sections[sec] = {}
+        elif sec and "=" in s and not s.startswith("#"):
+            k, v = s.split("=", 1)
+            sections[sec][k.strip()] = v.split("#", 1)[0].strip()
+    return {sec: {"ctx": kv.get("ctx-size", "-"),
+                  "ncpumoe": kv.get("n-cpu-moe", "-"),
+                  "spec": kv.get("spec-type", "-") or "-"}
+            for sec, kv in sections.items()}
+
+
+def run(base, key, model, prompt, n, temp, timeout):
+    payload = {"model": model, "messages": [{"role": "user", "content": prompt}],
+               "max_tokens": n, "temperature": temp, "stream": True,
+               "stream_options": {"include_usage": True},
+               # llama.cpp extension: makes the server embed its own timings in the stream →
+               # the table shows the SAME predicted_per_second as the llama.cpp web UI
+               # (without it we fall back to wall-clock estimates, which read lower).
+               "timings_per_token": True}
+    # ttft = first ANSWER token (delta.content) — for a thinking model that is the end of the think
+    # block, which is what the user waits for. t_first = first token of any kind (reasoning_content
+    # counts): the end of prefill, used for the wall-clock fallbacks. A thinking model that spends the
+    # whole gen budget reasoning never sets ttft → returned as None, rendered 'all-think' (NOT 0.00).
+    t0 = time.perf_counter(); ttft = None; t_first = None; ntok = 0; usage = None; timings = None
+    for raw in _open(base.rstrip("/") + "/chat/completions", key, payload, timeout):
+        line = raw.decode("utf-8", "ignore").strip()
+        if not line.startswith("data:"):
+            continue
+        body = line[5:].strip()
+        if body == "[DONE]":
+            break
+        try:
+            c = json.loads(body)
+        except ValueError:
+            continue
+        ch = c.get("choices") or [{}]
+        delta = ch[0].get("delta", {}) if ch else {}
+        if delta.get("content") or delta.get("reasoning_content"):
+            if t_first is None:
+                t_first = time.perf_counter() - t0
+            if delta.get("content") and ttft is None:
+                ttft = time.perf_counter() - t0
+            ntok += 1
+        if c.get("usage"):
+            usage = c["usage"]
+        if c.get("timings"):
+            timings = c["timings"]
+    total = time.perf_counter() - t0
+    comp = (usage or {}).get("completion_tokens") or ntok
+    ptok = (usage or {}).get("prompt_tokens")
+    gen_s = total - (t_first or total)
+    tg = comp / gen_s if gen_s > 0 else 0.0
+    pp = (ptok / t_first) if (ptok and t_first) else None
+    acc = None
+    if timings:  # llama.cpp's own numbers are authoritative when the router forwards them
+        tg = timings.get("predicted_per_second", tg)
+        pp = timings.get("prompt_per_second", pp)
+        ptok = timings.get("prompt_n", ptok)
+        dn, da = timings.get("draft_n"), timings.get("draft_n_accepted")
+        if dn:
+            acc = 100.0 * (da or 0) / dn
+    return {"tg": tg, "pp": pp, "ttft": ttft, "ptok": ptok or 0, "acc": acc}
+
+
+def wait_drain(drain_gb, window_s, label="", base=None, key=None):
+    """Force-unload resident models via the router API, then poll until VRAM used drops
+    below drain_gb (card is clean) or the window expires."""
+    used, _ = gpu_mem()
+    if base and used is not None and used > drain_gb * 1e9:
+        force_unload(base, key)
+    end = time.time() + window_s
+    announced = False
+    while used is not None and used > drain_gb * 1e9 and time.time() < end:
+        if not announced:
+            print(f"    … waiting for VRAM to drain before {label} ({gb(used)} used)", file=sys.stderr)
+            announced = True
+        time.sleep(5)
+        used, _ = gpu_mem()
+    return used
+
+
+def gb(b):
+    return f"{b / 1e9:.1f} GB" if b else "-"
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("-u", "--url", default="http://127.0.0.1:11343/v1")
+    ap.add_argument("-k", "--key", default="no-key-required")
+    ap.add_argument("-m", "--models", default="all", help="comma-separated ids, or 'all'")
+    ap.add_argument("-x", "--skip", default="", help="comma-separated ids to skip")
+    ap.add_argument("-n", "--tokens", type=int, default=256)
+    ap.add_argument("-t", "--temp", type=float, default=0.3)
+    ap.add_argument("--ctx", type=int, default=0, help="also record decode at ~this many prompt tokens")
+    ap.add_argument("--timeout", type=int, default=600, help="per-request stall timeout (s)")
+    ap.add_argument("--config", default=os.path.join(HERE, "config.ini"))
+    ap.add_argument("--out", default=os.path.join(HERE, "bench-results.md"))
+    ap.add_argument("--history", default=os.path.join(HERE, "bench-history.md"))
+    ap.add_argument("--retries", type=int, default=4, help="retries while a model is still mounting")
+    ap.add_argument("--retry-wait", type=int, default=15, help="seconds between retries")
+    ap.add_argument("--drain-gb", type=float, default=5.5,
+                    help="VRAM considered 'clean' below this (the desktop alone uses 3.5–4.5 GB here)")
+    ap.add_argument("--min-free-gb", type=float, default=0.8,
+                    help="flag a model that leaves less VRAM free than this after loading (busy-desktop risk)")
+    ap.add_argument("--max-gtt-gb", type=float, default=2.0,
+                    help="(inert on NVIDIA — kept for fl parity)")
+    ap.add_argument("--unload-wait", type=int, default=360,
+                    help="after a too-tight model, wait up to this long for it to unload before continuing "
+                         "(sleep-idle-seconds keeps models resident up to 120s here)")
+    ap.add_argument("--no-vram-ok", action="store_true",
+                    help="allow a multi-model sweep even though nvidia-smi (and thus the guards) is unavailable")
+    a = ap.parse_args()
+
+    models = list_models(a.url, a.key) if a.models == "all" else [m.strip() for m in a.models.split(",")]
+    skip = {s.strip() for s in a.skip.split(",") if s.strip()}
+    models = [m for m in models if m not in skip]
+    cfg = parse_config(a.config)
+
+    used0, total_vram = gpu_mem("vram")
+    if total_vram is None and len(models) > 1 and not a.no_vram_ok:
+        sys.exit("ABORT: nvidia-smi is not readable here, so the VRAM guards can't work.\n"
+                 "Run this ON the GPU box, bench a single model with -m <id>, or pass --no-vram-ok to accept the risk.")
+    # The BASELINE must be the desktop, not a resident model: the 2026-10-19 sweep started while pi's
+    # gemma was still mounted and recorded "baseline 7.5 GB" for a ~4 GB desktop. Drain first (force-
+    # unload + poll, same as before every model), record the result; keep the pre-drain figure so the
+    # header says which it was.
+    resident0 = None
+    if used0 is not None and used0 > a.drain_gb * 1e9:
+        resident0 = used0
+        used0 = wait_drain(a.drain_gb, a.unload_wait, "the baseline", a.url, a.key)
+        if used0 is not None and used0 > a.drain_gb * 1e9:
+            print(f"⚠ baseline VRAM used is {gb(used0)} with nothing of ours resident — desktop apps "
+                  f"(browser/electron/game) hold it. Margins below are for THIS desktop; ideally close "
+                  f"GPU apps and re-run (or raise --drain-gb if this is normal).", file=sys.stderr)
+
+    task = "Write a Python function that merges two sorted lists, with a short docstring and one example."
+    filler = ("The quick brown fox jumps over the lazy dog. " * max(1, a.ctx // 9)) if a.ctx else None
+
+    cols = ["model", "cfg ctx", "n-cpu-moe", "spec", "decode t/s", "prefill t/s", "TTFT s", "VRAM used", "VRAM free"]
+    if a.ctx:
+        cols.append(f"decode@{a.ctx // 1000}k")
+
+    latest = open(a.out, "w", encoding="utf-8")
+    hist = open(a.history, "a", encoding="utf-8")
+    baseline = gb(used0)
+    if resident0 is not None and used0 is not None and resident0 - used0 > 0.5e9:
+        baseline += f" (was {gb(resident0)} with a resident model, unloaded first)"
+    meta = (f"_Run {datetime.now():%Y-%m-%d %H:%M} · server `{a.url}` · VRAM total {gb(total_vram)} · "
+            f"baseline used {baseline} · gen {a.tokens} tok · guard: free≥{a.min_free_gb} GB_")
+
+    def emit(line, both=True):
+        print(line)
+        latest.write(line + "\n"); latest.flush()
+        if both:
+            hist.write(line + "\n"); hist.flush()
+
+    latest.write("# llama.cpp benchmark results (latest run)\n\n")
+    hist.write(f"\n## run {datetime.now():%Y-%m-%d %H:%M}\n\n")
+    emit(meta + "\n")
+    emit("| " + " | ".join(cols) + " |")
+    emit("|" + "|".join(["---"] * len(cols)) + "|")
+
+    for m in models:
+        c = cfg.get(m, {})
+        base_cells = [m, c.get("ctx", "-"), c.get("ncpumoe", "-"), c.get("spec", "-")]
+        # A CLEAN card before EVERY load (incl. the first): force-unload via the router API,
+        # then verify with nvidia-smi — double residency = bogus numbers (or a failed load).
+        u = wait_drain(a.drain_gb, a.unload_wait, m, a.url, a.key)
+        if u is not None and u > a.drain_gb * 1e9:
+            emit("")
+            emit(f"_⚠ aborted before {m}: VRAM still {gb(u)} used after {a.unload_wait}s — something "
+                 f"won't unload (leftover model / GPU apps). If that's your normal desktop, raise --drain-gb._")
+            print(f"\n⚠ aborted: card not clean ({gb(u)} used).", file=sys.stderr)
+            break
+        try:
+            # warmup — retries wait out the mount; 'failed to load' aborts immediately
+            with_retries(lambda: run(a.url, a.key, m, "hi", 8, a.temp, a.timeout), a.retries, a.retry_wait, m)
+            used, _ = gpu_mem("vram")
+            free = (total_vram - used) if (total_vram and used) else None
+            reason = None
+            if free is not None and free < a.min_free_gb * 1e9:
+                reason = f"only {gb(free)} VRAM free"
+            if reason:
+                # Too tight — do NOT generate (allocates more). Record the row; the sweep continues
+                # once this model has fully unloaded (see below).
+                hint = ("raise n-cpu-moe" if c.get("ncpumoe", "-") != "-"
+                        else "lower ctx / n-gpu-layers / smaller quant")
+                cells = base_cells + [f"⚠ {reason} — SKIPPED ({hint})", "", "", gb(used), gb(free)]
+                if a.ctx:
+                    cells.append("")
+            else:
+                r = with_retries(lambda: run(a.url, a.key, m, task, a.tokens, a.temp, a.timeout),
+                                 a.retries, a.retry_wait, m)
+                if r.get("acc") is not None:
+                    base_cells[3] = f"{base_cells[3]} ({r['acc']:.0f}% acc)"
+                cells = base_cells + [f"{r['tg']:.1f}", (f"{r['pp']:.0f}" if r['pp'] else "-"),
+                                      (f"{r['ttft']:.2f}" if r['ttft'] is not None else "all-think"),
+                                      gb(used), gb(free)]
+                if a.ctx:
+                    rc = with_retries(lambda: run(a.url, a.key, m, filler + "\n\n" + task, a.tokens, a.temp, a.timeout),
+                                      a.retries, a.retry_wait, m)
+                    cells.append(f"{rc['tg']:.1f}")
+        except Exception as e:
+            err = f"HTTP {e.code} (check server log)" if isinstance(e, urllib.error.HTTPError) else f"ERROR {e}"
+            cells = base_cells + [err] + [""] * (len(cols) - len(base_cells) - 1)
+        emit("| " + " | ".join(str(x) for x in cells) + " |")
+        # (no post-row wait needed: the pre-load drain above protects the next iteration,
+        #  including after ⚠-tight rows where generation was skipped)
+
+    latest.write("\n_Tuning hints: **VRAM free** = headroom to raise `ctx-size` or lower `n-cpu-moe` "
+                 "(more experts on GPU → faster). Low decode t/s on an offloaded MoE → lower `n-cpu-moe` "
+                 "if free allows, or pick a smaller/K-quant file (CPU experts are bandwidth-bound). "
+                 "High **baseline used** invalidates the run. For spec/MTP models the acceptance % is in the spec column. "
+                 "**TTFT** = time to the first ANSWER token (for thinking models: the think block); `all-think` = the whole "
+                 "gen budget was reasoning and no answer arrived (decode t/s is still valid — raise `-n` or use the "
+                 "-low/-instant alias). Prefill t/s at the ~30-token prompt is fixed-overhead noise; use `--ctx` for a real number._\n")
+    latest.close(); hist.close()
+    print(f"\nwrote {a.out} (+ appended {a.history})")
+
+
+if __name__ == "__main__":
+    main()