[Sync] Lua Hyprland chain, licence cleanup

fl/.config/vllm/vllmctl.py

@@ -0,0 +1,398 @@
+#!/usr/bin/env python3
+"""
+vllmctl — run vLLM (docker, ROCm) on fl WITHOUT freezing the box.
+
+The RX 7600 XT drives the display; VRAM overcommit spills to GTT and freezes the PC. vLLM grabs
+`gpu-memory-utilization` × total VRAM at startup and keeps it, so this script gates every start:
+
+  1. llama.service must be STOPPED (or --stop-router): a LAN client hitting :11343 would make the
+     router load a model on top of vLLM = the freeze. --unsafe-keep-router skips this gate.
+  2. the card must be clean (VRAM used ≤ drain-gb; resident router models are unloaded via its API).
+  3. projected free VRAM = total − used − budget must be ≥ reserve-gb (else abort, --force overrides).
+
+Presets live in vllm.ini next to this file (same key = --flag convention as llamacpp/config.ini).
+
+  ./vllmctl.py up <preset> [--stop-router] [--force] [--dry-run]   start, wait for /health
+  ./vllmctl.py down                                                stop, wait for VRAM to drain
+  ./vllmctl.py status | logs | pull | gpu-check | exec <cmd...>
+Stdlib only. Run ON fl.
+"""
+import argparse, grp, json, os, re, shlex, subprocess, sys, time, urllib.request, urllib.error
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+INI = os.path.join(HERE, "vllm.ini")
+DRM = "/sys/class/drm/renderD128/device"
+
+
+# ─── ini ────────────────────────────────────────────────────────────────────────────────────
+def parse_ini(path):
+    """{section: {key: value}} — inline `# comments` stripped, order kept."""
+    sections, sec = {}, None
+    for ln in open(path, encoding="utf-8").read().splitlines():
+        s = ln.strip()
+        if not s or s.startswith("#"):
+            continue
+        if s.startswith("[") and s.endswith("]"):
+            sec = s[1:-1]; sections[sec] = {}
+        elif sec and "=" in s:
+            k, v = s.split("=", 1)
+            sections[sec][k.strip()] = v.split(" #", 1)[0].split("\t#", 1)[0].strip()
+    return sections
+
+
+def build_args(preset):
+    """vllm serve args from a preset section (x-* and env.* are not flags)."""
+    args = []
+    for k, v in preset.items():
+        if k.startswith("x-") or k.startswith("env."):
+            continue
+        if v.lower() == "false":
+            continue
+        if v.lower() == "true":
+            args.append(f"--{k}")
+        else:
+            args += [f"--{k}", v]
+    return args
+
+
+# ─── gpu / router ───────────────────────────────────────────────────────────────────────────
+def sysfs(name):
+    try:
+        return int(open(os.path.join(DRM, name)).read().strip())
+    except (OSError, ValueError):
+        return None
+
+
+def vram():
+    """(used, total, gtt_used) bytes from amdgpu sysfs; None where unreadable."""
+    return sysfs("mem_info_vram_used"), sysfs("mem_info_vram_total"), sysfs("mem_info_gtt_used")
+
+
+def gb(b):
+    return f"{b / 1e9:.1f} GB" if b is not None else "-"
+
+
+def http(url, payload=None, timeout=10):
+    data = json.dumps(payload).encode() if payload is not None else None
+    hdrs = {"Content-Type": "application/json"} if data else {}
+    return urllib.request.urlopen(urllib.request.Request(url, data=data, headers=hdrs), timeout=timeout)
+
+
+def _status(m):
+    """Router /models status: newer builds send {'value': 'loaded'|'sleeping'|'unloaded'|…}, older a string."""
+    st = m.get("status", "loaded")
+    return st.get("value", "loaded") if isinstance(st, dict) else st
+
+
+def router_loaded(root):
+    """ids the llama.cpp router reports as loaded ([] if unreachable)."""
+    try:
+        d = json.load(http(root.rstrip("/") + "/models", timeout=5))
+    except Exception:
+        return []
+    return [m["id"] for m in d.get("data", []) if _status(m) in ("loaded", "loading")]
+
+
+def router_unload(root):
+    for m in router_loaded(root):
+        try:
+            http(root.rstrip("/") + "/models/unload", {"model": m}, timeout=30).read()
+        except Exception:
+            pass
+
+
+def router_active():
+    try:
+        r = subprocess.run(["systemctl", "is-active", "llama.service"], capture_output=True, text=True, timeout=10)
+    except (OSError, subprocess.TimeoutExpired):
+        return False
+    return r.stdout.strip() == "active"
+
+
+def router_hint():
+    return "" if router_active() else "\n  (llama.service is STOPPED → sudo systemctl start llama.service to bring the router back)"
+
+
+def wait_drain(drain_gb, window_s, label):
+    used, _, _ = vram()
+    end = time.time() + window_s
+    said = False
+    while used is not None and used > drain_gb * 1e9 and time.time() < end:
+        if not said:
+            print(f"  … waiting for VRAM to drain {label} ({gb(used)} used)"); said = True
+        time.sleep(5)
+        used, _, _ = vram()
+    return used
+
+
+# ─── docker ─────────────────────────────────────────────────────────────────────────────────
+def docker(*args, check=True, capture=True):
+    return subprocess.run(["docker", *args], check=check, capture_output=capture, text=True)
+
+
+def container_state(name):
+    r = docker("inspect", "-f", "{{.State.Status}}", name, check=False)
+    return r.stdout.strip() if r.returncode == 0 else None
+
+
+def gpu_flags():
+    flags = ["--device", "/dev/kfd", "--device", "/dev/dri", "--group-add", "video",
+             "--cap-add", "SYS_PTRACE", "--security-opt", "seccomp=unconfined", "--ipc=host"]
+    try:
+        flags += ["--group-add", str(grp.getgrnam("render").gr_gid)]   # Arch: /dev/dri/renderD128 is group render
+    except KeyError:
+        pass
+    return flags
+
+
+def run_cmd(g, name, preset, envs):
+    cmd = ["docker", "run", "-d", "--name", g["container"], *gpu_flags(),
+           "-p", f'{g["port"]}:8000',
+           "-v", f'{g["models-dir"]}:/models:ro',
+           "-v", f'{g["hf-cache"]}:/root/.cache/huggingface']
+    for k, v in envs.items():
+        cmd += ["-e", f"{k}={v}"]
+    cmd.append(g["image"])
+    cmd += ["--host", "0.0.0.0", "--port", "8000", "--served-model-name", preset.get("served-model-name", name)]
+    cmd += build_args({k: v for k, v in preset.items() if k != "served-model-name"})
+    return cmd
+
+
+ROOT_CAUSE = re.compile(r"hipError|HSA_STATUS|code object|gfx\d+|no kernel image|EngineCore.*(ERROR|Error|error)"
+                        r"|^\S*(ERROR|CRITICAL)|RuntimeError|ValueError|NotImplementedError|OutOfMemory|Killed|Segmentation"
+                        r"|free memory|not supported|Unsupported|Traceback|memory profiling|kv cache memory|weights take|peak", re.I)
+
+
+def root_cause(lines):
+    """The lines that explain a dead engine core: known ROCm/vLLM failure signatures, the FIRST
+    traceback's error line, and the last 8 lines — not the parent's 40-line stack trace."""
+    out, seen = [], set()
+    for l in lines:
+        if ROOT_CAUSE.search(l) and "api_server" not in l and "core_client" not in l and "launch_core_engines" not in l:
+            key = l.strip()[:120]
+            if key not in seen:
+                seen.add(key); out.append(l.rstrip())
+        if len(out) >= 30:
+            break
+    return out + ["…"] + [l.rstrip() for l in lines[-8:]]
+
+
+# ─── commands ───────────────────────────────────────────────────────────────────────────────
+def cmd_up(a, cfg):
+    g = cfg["global"]
+    if a.preset not in cfg or a.preset == "global":
+        sys.exit(f"unknown preset {a.preset!r}; have: {', '.join(k for k in cfg if k != 'global')}")
+    preset = dict(cfg[a.preset])
+    preset.update(dict(kv.split("=", 1) for kv in a.set))      # --set gpu-memory-utilization=0.73 --set max-model-len=8192
+    preset.setdefault("gpu-memory-utilization", g.get("gpu-memory-utilization", "0.75"))
+    envs = {k[4:]: v for k, v in g.items() if k.startswith("env.")}
+    envs.update({k[4:]: v for k, v in preset.items() if k.startswith("env.")})
+    envs.update(dict(e.split("=", 1) for e in a.env))
+    util = float(preset["gpu-memory-utilization"])
+    reserve = float(a.reserve if a.reserve is not None else g.get("reserve-gb", 2.0))
+    drain = float(g.get("drain-gb", 4.0))
+
+    cmd = run_cmd(g, a.preset, preset, envs)
+    print("docker command:\n  " + " ".join(shlex.quote(c) for c in cmd) + "\n")
+    if a.dry_run:
+        return
+
+    st = container_state(g["container"])
+    if st == "running":
+        sys.exit(f"container {g['container']} is already running — `vllmctl.py down` first")
+    if st:   # leftover from a crash: its log is the evidence, show the tail, then clear the name
+        r = docker("logs", "--tail", "15", g["container"], check=False)
+        print(f"removing old container ({st}); its last lines were:\n" + "\n".join("    | " + l for l in (r.stdout + r.stderr).strip().splitlines()[-15:]))
+        docker("rm", g["container"], check=False)
+
+    # 1. clean card — unload resident router models through its API while it is still up, then drain
+    if router_active():
+        router_unload(g["router"])
+    used = wait_drain(drain, 180, "before start")
+    used, total, gtt = vram()
+    if used is None or total is None:
+        if not a.force:
+            sys.exit(f"ABORT: cannot read {DRM}/mem_info_vram_* — no VRAM guard, not starting blind (--force overrides)")
+        print("⚠ FORCED without a VRAM guard (sysfs unreadable)"); used, total = 0, 1
+    if used > drain * 1e9 and not a.force:
+        sys.exit(f"ABORT: card not clean — {gb(used)} VRAM used (> drain-gb {drain}). Leftover model / GPU apps?")
+
+    # 2. projection — abort here costs nothing, llama.service is still running
+    budget = util * total
+    free_after = total - used - budget
+    weights = float(preset.get("x-weights-gb", 0) or 0) * 1e9
+    print(f"VRAM: total {gb(total)} · used now {gb(used)} (GTT {gb(gtt)}) · vLLM budget {util:.2f} × total = {gb(budget)}"
+          f"\n      → projected free {gb(free_after)} (reserve {reserve} GB)")
+    if weights:
+        room = budget - weights
+        print(f"      budget − weights {gb(weights)} = {gb(room)} for activations + KV cache"
+              + ("  ⚠ < 1 GB: expect 'no memory for KV cache' → raise util or lower max-model-len" if room < 1e9 else ""))
+    if free_after < reserve * 1e9:
+        suggest = max(0.0, (total - used - reserve * 1e9) / total)
+        msg = (f"projected free {gb(free_after)} < reserve {reserve} GB. Max safe gpu-memory-utilization right now "
+               f"= {suggest:.2f} (desktop is using {gb(used)}).")
+        if not a.force:
+            sys.exit("ABORT: " + msg + f"\n  → free the desktop (vramx) or `--set gpu-memory-utilization={suggest:.2f}` "
+                     f"(+ `--set max-model-len=8192` if weights leave < 1 GB); --force to start anyway.")
+        print("⚠ FORCED: " + msg)
+
+    # 3. router gate — the only thing that stops a LAN client from loading a model on top of vLLM
+    if router_active():
+        if a.stop_router:
+            print("stopping llama.service (sudo) …")
+            subprocess.run(["sudo", "systemctl", "stop", "llama.service"], check=True)
+        elif a.unsafe_keep_router:
+            print("⚠ llama.service stays ACTIVE: any client request to :11343 while vLLM runs = GTT freeze.")
+        else:
+            sys.exit("ABORT: llama.service is active. Any client request to :11343 would load a model on top "
+                     "of vLLM and freeze the PC.\n  → re-run with --stop-router (sudo systemctl stop llama.service), "
+                     "or --unsafe-keep-router if you really know nothing will call the router.")
+        used2, _, _ = vram()
+        if used2 is not None and used2 > used + 1e9:   # a client loaded something between projection and stop
+            print(f"⚠ VRAM grew to {gb(used2)} meanwhile — vLLM's own free-memory check should refuse; watch the start")
+
+    # 4. start + wait
+    cid = subprocess.run(cmd, check=True, capture_output=True, text=True).stdout.strip()
+    print(f"started {cid[:12]} — waiting for http://127.0.0.1:{g['port']}/health (timeout {g.get('health-timeout', 1800)} s; "
+          f"first start compiles kernels)")
+    end = time.time() + int(g.get("health-timeout", 1800))
+    last_log, shown = 0, set()
+    while time.time() < end:
+        st = container_state(g["container"])
+        if st != "running":
+            r = docker("logs", g["container"], check=False)
+            print("root cause candidates (full log: docker logs " + g["container"] + "):")
+            for l in root_cause((r.stdout + r.stderr).splitlines()):
+                print("    | " + l[:200])
+            sys.exit(f"ABORT: container is {st or 'gone'}. If a line names gfx1102 / 'code object' / hipErrorNoBinaryForGpu: "
+                     f"uncomment env.HSA_OVERRIDE_GFX_VERSION = 11.0.0 in [global] and retry." + router_hint())
+        try:
+            if http(f"http://127.0.0.1:{g['port']}/health", timeout=3).status == 200:
+                break
+        except Exception:
+            pass
+        if time.time() - last_log > 30:
+            r = docker("logs", "--tail", "3", g["container"], check=False)
+            tail = (r.stdout + r.stderr).strip().splitlines()[-3:]
+            for t in tail:
+                if t not in shown:                         # only lines not printed before (compile phases are quiet for minutes)
+                    print("    | " + t[:160]); shown.add(t)
+            last_log = time.time()
+        time.sleep(3)
+    else:
+        sys.exit("ABORT: no /health within the timeout — `vllmctl.py logs`, then `vllmctl.py down`" + router_hint())
+    used, total, gtt = vram()
+    print(f"UP. VRAM used {gb(used)} / free {gb(total - used)} · GTT {gb(gtt)}")
+    try:
+        ids = [m["id"] for m in json.load(http(f"http://127.0.0.1:{g['port']}/v1/models")).get("data", [])]
+        print(f"models: {', '.join(ids)}  →  http://192.168.0.204:{g['port']}/v1")
+    except Exception as e:
+        print(f"(/v1/models not readable yet: {e})")
+    print(f"next: ./bench-vs.py -t vllm:{preset.get('served-model-name', a.preset)}   ·   when done: ./vllmctl.py down")
+
+
+def cmd_down(a, cfg):
+    g = cfg["global"]
+    st = container_state(g["container"])
+    if st:
+        print(f"stopping {g['container']} ({st}) …")
+        docker("stop", "-t", "60", g["container"], check=False)
+        docker("rm", g["container"], check=False)
+    else:
+        print("container not running")
+    used = wait_drain(float(g.get("drain-gb", 4.0)), 120, "after stop")
+    used, total, gtt = vram()
+    print(f"VRAM used {gb(used)} / free {gb(total - used) if total and used is not None else '-'} · GTT {gb(gtt)}")
+    if not router_active():
+        print("llama.service is stopped → `sudo systemctl start llama.service` to bring the router back")
+
+
+def cmd_status(a, cfg):
+    g = cfg["global"]
+    used, total, gtt = vram()
+    print(f"container {g['container']}: {container_state(g['container']) or 'not running'}")
+    print(f"VRAM used {gb(used)} / total {gb(total)} · GTT {gb(gtt)}")
+    print(f"llama.service: {'ACTIVE' if router_active() else 'stopped'} · loaded: {router_loaded(g['router']) or '-'}")
+    try:
+        ids = [m["id"] for m in json.load(http(f"http://127.0.0.1:{g['port']}/v1/models", timeout=3)).get("data", [])]
+        print(f"vLLM :{g['port']} serving: {', '.join(ids)}")
+    except Exception:
+        print(f"vLLM :{g['port']}: not answering")
+
+
+def cmd_logs(a, cfg):
+    os.execvp("docker", ["docker", "logs", "-f", "--tail", "100", cfg["global"]["container"]])
+
+
+def cmd_pull(a, cfg):
+    os.execvp("docker", ["docker", "pull", cfg["global"]["image"]])
+
+
+GPU_CHECK = r"""
+import os, torch
+print('HSA_OVERRIDE_GFX_VERSION =', os.environ.get('HSA_OVERRIDE_GFX_VERSION', '(unset)'))
+print('cuda available:', torch.cuda.is_available())
+p = torch.cuda.get_device_properties(0)
+print('device:', torch.cuda.get_device_name(0) or '(no name)', p.gcnArchName, f'{p.total_memory / 1e9:.1f} GB')
+print('torch kernel …', end=' ', flush=True)   # a hard abort here (no code object for this arch) prints nothing more
+x = torch.ones(1024, device='cuda'); torch.cuda.synchronize(); print('ok', x.sum().item())
+import vllm._C  # noqa
+ops = sorted({s.name.split('::', 1)[1] for s in torch._C._jit_get_all_schemas() if s.name.startswith('_C::')})
+print(f'vllm _C ops registered: {len(ops)}')
+f16 = dict(device='cuda', dtype=torch.float16)
+probes = {'silu_and_mul': lambda: torch.ops._C.silu_and_mul(torch.empty(4, 8, **f16), torch.randn(4, 16, **f16)),
+          'gelu_fast':    lambda: torch.ops._C.gelu_fast(torch.empty(4, 8, **f16), torch.randn(4, 8, **f16)),
+          'rms_norm':     lambda: torch.ops._C.rms_norm(torch.empty(4, 8, **f16), torch.randn(4, 8, **f16), torch.ones(8, **f16), 1e-6)}
+for name, fn in probes.items():
+    if name in ops:
+        print(f'vllm custom op {name} …', end=' ', flush=True); fn(); torch.cuda.synchronize(); print('ok -> the image has kernels for this arch')
+        break
+else:
+    print('none of the probe ops exist in this build; first ops:', ops[:15])
+"""
+
+
+def cmd_gpu_check(a, cfg):
+    """Does the image see the card? Prints torch's device name + arch (needs no model)."""
+    g = cfg["global"]
+    envs = [x for k, v in g.items() if k.startswith("env.") for x in ("-e", f"{k[4:]}={v}")]
+    envs += [x for e in a.env for x in ("-e", e)]
+    code = GPU_CHECK
+    cmd = ["docker", "run", "--rm", *gpu_flags(), *envs, "--entrypoint", "python3", g["image"], "-u", "-c", code]
+    print(" ".join(shlex.quote(c) for c in cmd))
+    os.execvp("docker", cmd)
+
+
+def cmd_exec(a, cfg):
+    g = cfg["global"]
+    if container_state(g["container"]) == "running":
+        os.execvp("docker", ["docker", "exec", "-it", g["container"], *a.cmd])
+    envs = [x for k, v in g.items() if k.startswith("env.") for x in ("-e", f"{k[4:]}={v}")]
+    os.execvp("docker", ["docker", "run", "--rm", "-it", *gpu_flags(), *envs, "--entrypoint", a.cmd[0], g["image"], *a.cmd[1:]])
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--ini", default=INI)
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    up = sub.add_parser("up"); up.add_argument("preset")
+    up.add_argument("--stop-router", action="store_true", help="sudo systemctl stop llama.service first")
+    up.add_argument("--unsafe-keep-router", action="store_true", help="leave llama.service running (freeze risk)")
+    up.add_argument("--force", action="store_true", help="ignore the VRAM projection / clean-card checks")
+    up.add_argument("--reserve", type=float, help="override reserve-gb")
+    up.add_argument("--dry-run", action="store_true", help="print the docker command only")
+    up.add_argument("--env", action="append", default=[], metavar="KEY=VAL", help="extra container env (e.g. HSA_OVERRIDE_GFX_VERSION=11.0.0)")
+    up.add_argument("--set", action="append", default=[], metavar="KEY=VAL", help="override a preset key for this start (e.g. gpu-memory-utilization=0.73)")
+    sub.add_parser("down"); sub.add_parser("status"); sub.add_parser("logs"); sub.add_parser("pull")
+    gc = sub.add_parser("gpu-check"); gc.add_argument("--env", action="append", default=[], metavar="KEY=VAL")
+    ex = sub.add_parser("exec"); ex.add_argument("cmd", nargs=argparse.REMAINDER)
+    a = ap.parse_args()
+    cfg = parse_ini(a.ini)
+    if "global" not in cfg:
+        sys.exit(f"{a.ini}: no [global] section")
+    {"up": cmd_up, "down": cmd_down, "status": cmd_status, "logs": cmd_logs, "pull": cmd_pull,
+     "gpu-check": cmd_gpu_check, "exec": cmd_exec}[a.cmd](a, cfg)
+
+
+if __name__ == "__main__":
+    main()