hypertower/v4/scripts/analysis/summarize_run.py

"""summarize_run — print cross-rep stats for one v4 run folder.

A "run" is a folder like ``v4/results/experiments/tri_v1/grid/bcd75_cw1_nt25/``
containing ``rep00/``, ``rep01/``, ... — each with a per-rep ``summary.json``
under any ``out_dir_tags`` subdir (typically ``binary/`` or ``binary/ntower/``).

Usage:
    python -m v4.scripts.analysis.summarize_run <run_folder> [--per-rep] [--json]

Examples:
    python -m v4.scripts.analysis.summarize_run \
        v4/results/experiments/tri_v1/grid/bcd75_cw1_nt25
    python -m v4.scripts.analysis.summarize_run \
        v4/results/experiments/tri_v1/baseline_tri --per-rep
"""
from __future__ import annotations

import argparse
import json
import re
from pathlib import Path

import numpy as np


def find_rep_summaries(run_dir: Path) -> list[tuple[int, Path]]:
    """Return [(rep_idx, summary_path), ...] sorted by rep_idx."""
    out: list[tuple[int, Path]] = []
    for rep_dir in sorted(run_dir.glob("rep*")):
        if not rep_dir.is_dir():
            continue
        m = re.match(r"rep(\d+)$", rep_dir.name)
        if not m:
            continue
        summary = next(iter(rep_dir.rglob("summary.json")), None)
        if summary is not None:
            out.append((int(m.group(1)), summary))
    return out


def load_rep(summary_path: Path) -> dict:
    """Extract the fields we summarise from one rep's summary.json.

    Uses `primary_metric` if present (for regression runs e.g. neg_mse),
    falling back to AUC for legacy classification runs.
    """
    d = json.loads(summary_path.read_text())
    pm = d.get("primary_metric", "auc")
    return {
        "primary":    pm,
        "val_mean":   float(d.get(f"mean_val_{pm}",  d.get("mean_val_auc",  float("nan")))),
        "val_std":    float(d.get(f"std_val_{pm}",   d.get("std_val_auc",   float("nan")))),
        "test_mean":  float(d.get(f"mean_test_{pm}", d.get("mean_test_auc", float("nan")))),
        "test_std":   float(d.get(f"std_test_{pm}",  d.get("std_test_auc",  float("nan")))),
        "elapsed_s":  float(d.get("elapsed_s",       float("nan"))),
        "eval_stage": d.get("eval_stage", "?"),
    }


def summarise(run_dir: Path) -> dict:
    reps = find_rep_summaries(run_dir)
    if not reps:
        return {"run": str(run_dir), "n_reps": 0, "reps": []}
    rows = [(idx, load_rep(p)) for idx, p in reps]
    val   = np.array([r[1]["val_mean"]  for r in rows])
    test  = np.array([r[1]["test_mean"] for r in rows])
    elaps = np.array([r[1]["elapsed_s"] for r in rows])
    out = {
        "run":         str(run_dir),
        "n_reps":      len(rows),
        "eval_stage":  rows[0][1]["eval_stage"],
        "primary":     rows[0][1]["primary"],
        "val_mean":    float(np.mean(val)),
        "val_std":     float(np.std(val)),
        "val_min":     float(np.min(val)),
        "val_max":     float(np.max(val)),
        "test_mean":   float(np.mean(test)),
        "test_std":    float(np.std(test)),
        "test_min":    float(np.min(test)),
        "test_max":    float(np.max(test)),
        "elapsed_total_s": float(np.sum(elaps)) if not np.isnan(elaps).any() else None,
        "reps": [
            {"rep": idx, **info} for idx, info in rows
        ],
    }
    return out


def render(s: dict, per_rep: bool = False) -> str:
    if s["n_reps"] == 0:
        return f"Run: {s['run']}\n  no reps with summary.json found."

    metric = s.get("primary", "auc")
    lines = [
        f"Run:        {s['run']}",
        f"Reps:       {s['n_reps']}  (eval_stage={s['eval_stage']}, metric={metric})",
        f"Val  {metric}:   {s['val_mean']:.4f} ± {s['val_std']:.4f}   "
        f"[min={s['val_min']:.4f}  max={s['val_max']:.4f}]",
        f"Test {metric}:   {s['test_mean']:.4f} ± {s['test_std']:.4f}   "
        f"[min={s['test_min']:.4f}  max={s['test_max']:.4f}]",
    ]
    if s.get("elapsed_total_s") is not None:
        h = s["elapsed_total_s"] / 3600
        lines.append(f"Compute:    {s['elapsed_total_s']:.0f} s total  ({h:.1f} h)")

    if per_rep:
        lines.append("")
        lines.append("Per-rep breakdown:")
        lines.append(f"  {'rep':>4s}   {'val':>7s}   {'test':>7s}   {'elapsed':>7s}")
        for r in s["reps"]:
            elapsed = (f"{r['elapsed_s']:.0f}s" if not np.isnan(r['elapsed_s']) else "-")
            lines.append(
                f"  {r['rep']:>4d}   "
                f"{r['val_mean']:>7.4f}   "
                f"{r['test_mean']:>7.4f}   "
                f"{elapsed:>7s}"
            )
    return "\n".join(lines)


def main():
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("run_dir", type=Path, help="Path to a run folder (contains repNN/ subdirs)")
    ap.add_argument("--per-rep", action="store_true", help="Print one line per rep")
    ap.add_argument("--json",    action="store_true", help="Emit JSON instead of formatted text")
    args = ap.parse_args()

    if not args.run_dir.is_dir():
        raise SystemExit(f"Not a directory: {args.run_dir}")

    s = summarise(args.run_dir)
    if args.json:
        print(json.dumps(s, indent=2))
    else:
        print(render(s, per_rep=args.per_rep))


if __name__ == "__main__":
    main()