"""Post-training (no retraining) pruning of Gemma 4 E2B, measured by WikiText-2 perplexity.

Mirrors the comparisons in Frankle & Carbin (ICLR 2019) that make sense without training:
  - layerwise : prune the smallest-|w| fraction of every matrix separately (the paper's Lenet/Conv setting)
  - global    : one |w| threshold across all target matrices (the paper's VGG-19/Resnet-18 setting)
  - random    : prune a random fraction of every matrix (the paper's "random sparsity" control)

Targets: the attention (q, k, v, o) and MLP (gate, up, down) weight matrices of all 35 decoder layers,
as instantiated by mlx-lm (the KV-shared layers 15-34 have no k/v projections). Embeddings, the
per-layer-embedding (PLE) table and gates, norms and the tied LM head are not pruned.

Masks are nested across sparsity levels, so each method prunes cumulatively from low to high sparsity:
pruned weights are exactly zero and therefore stay the smallest.
"""
import argparse
import gc
import json
import math
import sys
import time

import mlx.core as mx
import numpy as np
import pyarrow.parquet as pq
from mlx_lm import load

N_CODES = 1 << 15  # |bf16| bit patterns (sign bit cleared) are ordered like the values they encode


def log(msg):
    print(time.strftime("%H:%M:%S"), msg, flush=True)


def peak_gb():
    f = getattr(mx, "get_peak_memory", None) or mx.metal.get_peak_memory
    return f() / 1e9


def clear_cache():
    (getattr(mx, "clear_cache", None) or mx.metal.clear_cache)()


def targets_of(model):
    out = []
    for i, layer in enumerate(model.layers):
        for name in ("q_proj", "k_proj", "v_proj", "o_proj"):
            if hasattr(layer.self_attn, name):
                out.append((f"layers.{i}.self_attn.{name}", getattr(layer.self_attn, name)))
        for name in ("gate_proj", "up_proj", "down_proj"):
            out.append((f"layers.{i}.mlp.{name}", getattr(layer.mlp, name)))
    for name, mod in out:
        assert mod.weight.dtype == mx.bfloat16 and mod.weight.ndim == 2, (name, mod.weight.dtype, mod.weight.shape)
    return out


def codes(w):
    return mx.view(w, mx.uint16) & 0x7FFF


def hist(w):
    return np.bincount(np.array(codes(w)).ravel(), minlength=N_CODES)


def cut(h, k):
    """Threshold code T and how many entries with code T must also go, so that exactly k are pruned."""
    if k <= 0:
        return 0, 0
    cs = np.cumsum(h)
    t = int(np.searchsorted(cs, k))
    below = int(cs[t - 1]) if t > 0 else 0
    return t, k - below


def uniform(shape, seed):
    return mx.random.uniform(shape=shape, key=mx.random.key(seed))


def tied_u(w, t, seed):
    """Tie-breaking random numbers of the entries whose magnitude code equals t."""
    return np.array(uniform(w.shape, seed))[np.array(codes(w)) == t]


def kth_smallest(us, m):
    """Tie entries with u <= this value are pruned: exactly the m with the smallest u."""
    if m <= 0:
        return -1.0
    return float(np.partition(np.concatenate(us), m - 1)[m - 1])


def prune(targets, method, sparsity, seed):
    tie_seed = lambda j: seed * 1000 + 500 + j
    if method == "global":
        t_glob, m_glob = cut(sum(hist(m.weight) for _, m in targets), round(sparsity * sum(m.weight.size for _, m in targets)))
        u_glob = kth_smallest([tied_u(m.weight, t_glob, tie_seed(j)) for j, (_, m) in enumerate(targets)], m_glob)
    for j, (_, m) in enumerate(targets):
        w = m.weight
        if method == "random":
            drop = uniform(w.shape, seed * 1000 + j) < sparsity
        else:
            if method == "layerwise":
                t, mt = cut(hist(w), round(sparsity * w.size))
                u_thr = kth_smallest([tied_u(w, t, tie_seed(j))], mt)
            else:
                t, u_thr = t_glob, u_glob
            c = codes(w)
            drop = (c < t) | ((c == t) & (uniform(w.shape, tie_seed(j)) <= u_thr))
        m.weight = mx.where(drop, mx.zeros_like(w), w)
        mx.eval(m.weight)


def zero_fraction(targets):
    z = sum(mx.sum(m.weight == 0).item() for _, m in targets)
    n = sum(m.weight.size for _, m in targets)
    return z / n


def windows_of(tokenizer, path, seq_len, n_windows):
    text = "\n\n".join(pq.read_table(path).column("text").to_pylist())
    ids = tokenizer.encode(text, add_special_tokens=False)
    bos = tokenizer.bos_token_id
    step = seq_len - 1  # every window is [BOS] + seq_len-1 text tokens; all text tokens are predicted
    wins = [[bos] + ids[i:i + step] for i in range(0, len(ids) - step + 1, step)]
    log(f"WikiText-2 test: {len(ids):,} tokens -> {len(wins)} windows of {seq_len}; using {n_windows or len(wins)}")
    return wins[:n_windows] if n_windows else wins


def evaluate(model, wins, chunk=256):
    nll, n, per_window, t0 = 0.0, 0, [], time.time()
    for win in wins:
        x = mx.array(win)[None]
        logits = model(x[:, :-1])  # bf16; the float32 copy is made per chunk to keep memory low
        mx.eval(logits)
        y = x[:, 1:]
        s = 0.0
        for c in range(0, y.shape[1], chunk):
            lc, yc = logits[:, c:c + chunk].astype(mx.float32), y[:, c:c + chunk]
            part = mx.sum(mx.logsumexp(lc, axis=-1) - mx.take_along_axis(lc, yc[..., None], axis=-1).squeeze(-1))
            mx.eval(part)
            s += part.item()
        del logits
        nll += s
        n += y.size
        per_window.append(round(s / y.size, 5))
    mean = nll / n
    return {"nll": mean, "ppl": math.exp(mean) if mean < 700 else float("inf"), "tokens": n,
            "eval_s": round(time.time() - t0, 1), "window_nll": per_window}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--model", default="google/gemma-4-E2B")
    ap.add_argument("--data", required=True)
    ap.add_argument("--seq-len", type=int, default=1024)
    ap.add_argument("--windows", type=int, default=64, help="0 = whole test set")
    ap.add_argument("--methods", default="layerwise,global,random")
    ap.add_argument("--sparsities", default="0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9")
    ap.add_argument("--seed", type=int, default=0)
    ap.add_argument("--out", required=True)
    ap.add_argument("--smoke", action="store_true", help="dense + layerwise 50%% on 2 windows, then exit")
    ap.add_argument("--resume", action="store_true", help="skip evaluations already in --out (pruning is still replayed)")
    ap.add_argument("--stop-ratio", type=float, default=0,
                    help="after a level whose perplexity exceeds this multiple of dense, skip that method's higher levels (0 = never)")
    a = ap.parse_args()

    # Do not keep freed buffers: on a 16 GB machine the cache pushes the model into swap.
    (getattr(mx, "set_cache_limit", None) or mx.metal.set_cache_limit)(0)
    model, tok = load(a.model)
    targets = targets_of(model)
    n_params = sum(m.weight.size for _, m in targets)
    log(f"loaded {a.model}; {len(targets)} target matrices, {n_params:,} prunable weights; peak {peak_gb():.1f} GB")
    wins = windows_of(tok, a.data, a.seq_len, 2 if a.smoke else a.windows)

    meta = {"model": a.model, "seq_len": a.seq_len, "windows": len(wins), "targets": len(targets), "prunable_weights": n_params, "seed": a.seed}
    done = {}  # (method, sparsity) -> ppl of evaluations already in --out
    if a.resume:
        try:
            for line in open(a.out):
                r = json.loads(line)
                if all(r.get(k) == v for k, v in meta.items()):
                    done[(r["method"], r["sparsity"])] = r["ppl"]
        except FileNotFoundError:
            pass
        log(f"resume: {len(done)} evaluations already in {a.out}: {sorted(done)}")
    out = open(a.out, "a")

    def record(method, sparsity, actual, res):
        row = {**meta, "method": method, "sparsity": sparsity, "actual_sparsity": round(actual, 6), **res, "peak_gb": round(peak_gb(), 2)}
        out.write(json.dumps(row) + "\n")
        out.flush()
        log(f"{method:9s} s={sparsity:.2f} (actual {actual:.4f})  ppl={row['ppl']:.3f}  nll={row['nll']:.4f}  [{row['eval_s']}s, peak {row['peak_gb']} GB]")
        return row["ppl"]

    if ("dense", 0.0) not in done:
        done[("dense", 0.0)] = record("dense", 0.0, zero_fraction(targets), evaluate(model, wins))
    stop_at = a.stop_ratio * done[("dense", 0.0)] if a.stop_ratio else float("inf")
    if a.smoke:
        t0 = time.time()
        prune(targets, "layerwise", 0.5, a.seed)
        log(f"pruning layerwise 50% took {time.time() - t0:.1f}s")
        record("layerwise", 0.5, zero_fraction(targets), evaluate(model, wins))
        return

    levels = [float(s) for s in a.sparsities.split(",")]
    for i, method in enumerate(a.methods.split(",")):
        if i > 0:  # restore the original weights
            del model, targets
            gc.collect()
            clear_cache()
            model, _ = load(a.model)
            targets = targets_of(model)
        for s in levels:
            t0 = time.time()
            prune(targets, method, s, a.seed)
            clear_cache()
            log(f"pruned {method} to {s:.2f} in {time.time() - t0:.1f}s")
            if (method, s) in done:
                log(f"{method} s={s:.2f}: already in results, skipping evaluation")
                ppl = done[(method, s)]
            else:
                ppl = record(method, s, zero_fraction(targets), evaluate(model, wins))
            if ppl > stop_at:
                log(f"{method}: ppl {ppl:.1f} exceeds {a.stop_ratio:g}x dense ({stop_at:.1f}); skipping its higher sparsities")
                break
    log("done")


if __name__ == "__main__":
    sys.exit(main())
