Add new regression and ensemble experiment configurations for V2-M and OrthoBridge

- Introduced multiple regression experiment configurations targeting vf_md, including:
  - cd_solo_reg_set.json: CD tower only regression setup.
  - img_solo_reg_set.json: Image tower only regression setup.
  - reg_head_epoch_sweep.json: Baseline regression sweeps at different epochs (50, 75, 100).
  - reg_head_set.json: Various regression setups including baseline and OrthoBridge configurations.
  - single_eye_reg.json: Single-eye regression setup for worst-eye aggregation analysis.

- Added ensemble configurations for OrthoBridge with different inner bridges:
  - ortho_alts_ensemble.json: Ensemble tests with ConcatBridge, PairwiseAdditiveBridge, and GatedAdditiveBridge.
  - ortho_alts_tritower.json: Tritower tests with the same inner bridges.

- Created V2-M specific configurations:
  - baseline_reg_nt50.json: Regression baseline with V2-M backbone.
  - geom_vec_gt.json and geom_vec_unet.json: Geometry vector injection experiments with V2-M.
  - single_l1_bridges.json: Single-eye ensemble experiments with various bridge types.
  - tritower_geom_gt.json: Tritower setup with GT contour-rasterized masks.

- Promoted existing experiments to higher repetitions for robustness.

v4/scripts/analysis/calibrate_regression.py

@@ -0,0 +1,173 @@
+"""Post-hoc regression calibration on predictions.h5 files.
+
+For each fold of a regression run, fit a linear calibration
+   actual_md ≈ a · predicted_md + b
+on the *val* split, then apply (a, b) to that fold's *test* predictions
+and report metrics before vs after calibration. No retraining required.
+
+Usage:
+    python -m v4.scripts.analysis.calibrate_regression \\
+        v4/results/experiments/reg_head/baseline_reg_nt50
+
+Pass a single predictions.h5 file or a folder; the script finds every
+predictions.h5 under it and produces a per-rep + aggregate report.
+"""
+from __future__ import annotations
+
+import argparse
+import math
+from pathlib import Path
+
+import h5py
+import numpy as np
+
+BIN_LABELS = ["<=-10", "-9..-5", "-4..-1", ">=1"]
+
+
+def _severity_bin(values: np.ndarray) -> np.ndarray:
+    bins = np.full(values.shape, -1, dtype=int)
+    bins[values <= -9.5] = 0
+    bins[(values > -9.5) & (values <= -4.5)] = 1
+    bins[(values > -4.5) & (values <= 0.0)] = 2
+    bins[values > 0.0] = 3
+    return bins
+
+
+def _decode(arr) -> np.ndarray:
+    return np.array([s.decode("utf-8") if isinstance(s, bytes) else str(s) for s in arr])
+
+
+def _metrics(y_true: np.ndarray, y_pred: np.ndarray) -> dict:
+    res = y_pred - y_true
+    mse = float(np.mean(res ** 2))
+    abs_res = np.abs(res)
+    out = {
+        "n":        int(y_true.size),
+        "mse":      mse,
+        "rmse":     math.sqrt(mse),
+        "mae":      float(np.mean(abs_res)),
+        "bias":     float(np.mean(res)),
+        "within_1": float(np.mean(abs_res <= 1.0)),
+        "within_3": float(np.mean(abs_res <= 3.0)),
+        "within_5": float(np.mean(abs_res <= 5.0)),
+        "r":        float(np.corrcoef(y_true, y_pred)[0, 1])
+                    if y_true.size > 1 and np.std(y_pred) > 0 else float("nan"),
+    }
+    bins_true = _severity_bin(y_true)
+    bins_pred = _severity_bin(y_pred)
+    valid = (bins_true >= 0) & (bins_pred >= 0)
+    out["bin_exact"]    = float(np.mean(bins_true[valid] == bins_pred[valid])) if valid.any() else float("nan")
+    out["bin_adjacent"] = float(np.mean(np.abs(bins_true[valid] - bins_pred[valid]) <= 1)) if valid.any() else float("nan")
+    return out
+
+
+def _fit_linear(y_true: np.ndarray, y_pred: np.ndarray) -> tuple[float, float]:
+    """Fit y_true = a * y_pred + b via OLS. Returns (a, b)."""
+    if y_pred.size < 2 or np.std(y_pred) == 0:
+        return 1.0, 0.0
+    a, b = np.polyfit(y_pred, y_true, 1)
+    return float(a), float(b)
+
+
+def calibrate_one(path: Path) -> dict:
+    """Run val→test linear calibration on one predictions.h5. Returns aggregate metrics."""
+    with h5py.File(path, "r") as f:
+        # Pick the last phase that has a head (heuristic: highest stage)
+        phases = sorted(k for k in f.keys() if isinstance(f[k], h5py.Group) and "logits" in f[k])
+        if not phases:
+            return {"path": str(path), "skipped": "no logits"}
+        # Prefer 'hb' if present
+        phase = "hb" if "hb" in phases else phases[-1]
+        grp = f[phase]
+        logits   = grp["logits"][:]            # (folds, epochs, samples, heads, outputs)
+        y_true   = grp["y_true"][:].astype(float)
+        split    = grp["split"][:]
+        n_folds, n_epochs, n_samples, n_heads, _ = logits.shape
+        # Use the last epoch and head 0, output 0
+        ep, head, out = n_epochs - 1, n_heads - 1, 0
+
+        raw_test:   list[np.ndarray] = []
+        cal_test:   list[np.ndarray] = []
+        truth_test: list[np.ndarray] = []
+        slopes: list[float] = []
+        intercepts: list[float] = []
+
+        for fold in range(n_folds):
+            labels = _decode(split[fold])
+            v_mask = (labels == "val")  & np.isfinite(y_true) & np.isfinite(logits[fold, ep, :, head, out])
+            t_mask = (labels == "test") & np.isfinite(y_true) & np.isfinite(logits[fold, ep, :, head, out])
+            if not v_mask.any() or not t_mask.any():
+                continue
+            v_pred = logits[fold, ep, v_mask, head, out].astype(float)
+            v_true = y_true[v_mask]
+            a, b = _fit_linear(v_true, v_pred)
+            slopes.append(a); intercepts.append(b)
+            t_pred = logits[fold, ep, t_mask, head, out].astype(float)
+            t_true = y_true[t_mask]
+            raw_test.append(t_pred)
+            cal_test.append(a * t_pred + b)
+            truth_test.append(t_true)
+
+    if not raw_test:
+        return {"path": str(path), "skipped": "no val/test rows"}
+
+    truth = np.concatenate(truth_test)
+    raw   = _metrics(truth, np.concatenate(raw_test))
+    cal   = _metrics(truth, np.concatenate(cal_test))
+    return {
+        "path":   str(path),
+        "slopes":     slopes,
+        "intercepts": intercepts,
+        "raw":    raw,
+        "cal":    cal,
+    }
+
+
+def _fmt(row: dict, label: str) -> str:
+    return (f"  {label}: mse={row['mse']:.3f} mae={row['mae']:.3f} bias={row['bias']:+.3f} "
+            f"r={row['r']:.3f}  bin_exact={row['bin_exact']:.3f} bin_adj={row['bin_adjacent']:.3f}  "
+            f"within_1={row['within_1']:.2f} within_3={row['within_3']:.2f}")
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("path", type=Path, help="predictions.h5 file or directory")
+    args = ap.parse_args()
+
+    if args.path.is_file():
+        files = [args.path]
+    else:
+        files = sorted(args.path.rglob("predictions.h5"))
+    if not files:
+        raise SystemExit(f"No predictions.h5 found under {args.path}")
+
+    # Aggregate raw vs cal across all reps
+    agg_raw_truth, agg_raw_pred = [], []
+    agg_cal_truth, agg_cal_pred = [], []
+    all_slopes, all_intercepts = [], []
+
+    for fp in files:
+        res = calibrate_one(fp)
+        if res.get("skipped"):
+            print(f"\n{fp.parent.parent.name}: SKIPPED ({res['skipped']})")
+            continue
+        rel = fp.relative_to(args.path) if args.path.is_dir() else fp.name
+        print(f"\n{rel}")
+        print(f"  fold-wise slopes:      {', '.join(f'{a:.2f}' for a in res['slopes'])}")
+        print(f"  fold-wise intercepts:  {', '.join(f'{b:+.2f}' for b in res['intercepts'])}")
+        print(_fmt(res["raw"], "raw "))
+        print(_fmt(res["cal"], "cal "))
+        all_slopes.extend(res["slopes"])
+        all_intercepts.extend(res["intercepts"])
+
+    if all_slopes:
+        print(
+            f"\nCalibration parameters across all reps × folds (n={len(all_slopes)}):\n"
+            f"  slope:      mean={np.mean(all_slopes):.3f} ± {np.std(all_slopes):.3f}  "
+            f"min={np.min(all_slopes):.2f}  max={np.max(all_slopes):.2f}\n"
+            f"  intercept:  mean={np.mean(all_intercepts):+.3f} ± {np.std(all_intercepts):.3f}"
+        )
+
+
+if __name__ == "__main__":
+    main()

v4/scripts/analysis/plot_regression_predictions.py

@@ -0,0 +1,349 @@
+"""Plot regression predictions against actual values from v4 predictions.h5.
+
+Usage:
+    python -m v4.scripts.analysis.plot_regression_predictions \
+        v4/results/experiments/reg_head/baseline_reg
+
+    python -m v4.scripts.analysis.plot_regression_predictions \
+        v4/results/experiments/reg_head/baseline_reg/rep00/binary/predictions.h5 \
+        --split val --phase hb --head hb_head --glaucoma-only
+"""
+from __future__ import annotations
+
+import argparse
+import math
+from pathlib import Path
+
+import h5py
+import matplotlib.pyplot as plt
+import numpy as np
+
+
+def _decode_array(arr: np.ndarray) -> np.ndarray:
+    return np.array([v.decode() if isinstance(v, bytes) else str(v) for v in arr])
+
+
+def _head_names(grp: h5py.Group) -> list[str]:
+    return [v.decode() if isinstance(v, bytes) else str(v) for v in grp["head_names"][:]]
+
+
+def _find_prediction_files(path: Path) -> list[Path]:
+    if path.is_file():
+        if path.name != "predictions.h5":
+            raise SystemExit(f"Expected predictions.h5 file, got: {path}")
+        return [path]
+    if not path.is_dir():
+        raise SystemExit(f"Not a file or directory: {path}")
+    files = sorted(path.rglob("predictions.h5"))
+    if not files:
+        raise SystemExit(f"No predictions.h5 files found under: {path}")
+    return files
+
+
+def _choose_phase(f: h5py.File, phase: str | None) -> str:
+    phases = sorted(k for k in f.keys() if isinstance(f[k], h5py.Group) and "logits" in f[k])
+    if not phases:
+        raise ValueError("No phase groups containing a logits dataset were found")
+    if phase is None:
+        return phases[0]
+    if phase not in phases:
+        raise ValueError(f"Phase {phase!r} not found. Available phases: {', '.join(phases)}")
+    return phase
+
+
+def _choose_head(grp: h5py.Group, head: str | None) -> tuple[str, int]:
+    names = _head_names(grp)
+    if not names:
+        raise ValueError("No head names found")
+    if head is None:
+        return names[0], 0
+    if head not in names:
+        raise ValueError(f"Head {head!r} not found. Available heads: {', '.join(names)}")
+    return head, names.index(head)
+
+
+def _collect_points(
+    path: Path,
+    *,
+    phase: str | None,
+    head: str | None,
+    split: str,
+    epoch: int,
+    output_index: int,
+    glaucoma_only: bool,
+) -> tuple[np.ndarray, np.ndarray, np.ndarray, str, str]:
+    with h5py.File(path, "r") as f:
+        phase_name = _choose_phase(f, phase)
+        grp = f[phase_name]
+        head_name, head_idx = _choose_head(grp, head)
+
+        logits = grp["logits"]
+        n_folds, n_epochs, n_samples, _, n_outputs = logits.shape
+        epoch_idx = epoch if epoch >= 0 else n_epochs + epoch
+        if not 0 <= epoch_idx < n_epochs:
+            raise ValueError(f"Epoch {epoch} is out of range for {n_epochs} epochs")
+        if not 0 <= output_index < n_outputs:
+            raise ValueError(f"Output index {output_index} is out of range for {n_outputs} outputs")
+
+        y_true_all = grp["y_true"][:].astype(float)
+        split_all = grp["split"][:]
+
+        actuals: list[np.ndarray] = []
+        preds: list[np.ndarray] = []
+        folds: list[np.ndarray] = []
+        for fold in range(n_folds):
+            labels = _decode_array(split_all[fold])
+            mask = np.ones(n_samples, dtype=bool) if split == "all" else labels == split
+            if glaucoma_only:
+                mask &= y_true_all != 0
+            y_pred = logits[fold, epoch_idx, :, head_idx, output_index].astype(float)
+            finite = mask & np.isfinite(y_true_all) & np.isfinite(y_pred)
+            if not finite.any():
+                continue
+            actuals.append(y_true_all[finite])
+            preds.append(y_pred[finite])
+            folds.append(np.full(int(finite.sum()), fold, dtype=int))
+
+    if not actuals:
+        suffix = " and y_true != 0" if glaucoma_only else ""
+        raise ValueError(f"No finite rows found for split={split!r}{suffix}")
+    return (
+        np.concatenate(actuals),
+        np.concatenate(preds),
+        np.concatenate(folds),
+        phase_name,
+        head_name,
+    )
+
+
+def _metrics(y_true: np.ndarray, y_pred: np.ndarray) -> dict[str, float]:
+    residual = y_pred - y_true
+    mse = float(np.mean(residual ** 2))
+    abs_residual = np.abs(residual)
+    out = {
+        "n": float(y_true.size),
+        "mse": mse,
+        "rmse": math.sqrt(mse),
+        "mae": float(np.mean(abs_residual)),
+        "bias": float(np.mean(residual)),
+        "within_1": float(np.mean(abs_residual <= 1.0)),
+        "within_3": float(np.mean(abs_residual <= 3.0)),
+        "within_5": float(np.mean(abs_residual <= 5.0)),
+    }
+    if y_true.size > 1 and np.std(y_true) > 0 and np.std(y_pred) > 0:
+        out["r"] = float(np.corrcoef(y_true, y_pred)[0, 1])
+    else:
+        out["r"] = float("nan")
+    return out
+
+
+BIN_LABELS = ["<=-10", "-9..-5", "-4..-1", ">=1"]
+
+
+def _severity_bin(values: np.ndarray) -> np.ndarray:
+    """Map continuous values onto glaucoma severity bins.
+
+    Actual labels use integer bins: <=-10, -9..-5, -4..-1, >=1.
+    Predictions are continuous, so boundaries are placed halfway between
+    adjacent integer bins: -9.5, -4.5, and 0.0.
+    """
+    bins = np.full(values.shape, -1, dtype=int)
+    bins[values <= -9.5] = 0
+    bins[(values > -9.5) & (values <= -4.5)] = 1
+    bins[(values > -4.5) & (values <= 0.0)] = 2
+    bins[values > 0.0] = 3
+    return bins
+
+
+def _bin_summary_lines(y_true: np.ndarray, y_pred: np.ndarray) -> list[str]:
+    actual_bins = _severity_bin(y_true)
+    pred_bins = _severity_bin(y_pred)
+    valid = (actual_bins >= 0) & (pred_bins >= 0)
+    if not valid.any():
+        return ["Bin summary: no rows matched the configured severity bins"]
+
+    actual_bins = actual_bins[valid]
+    pred_bins = pred_bins[valid]
+    confusion = np.zeros((len(BIN_LABELS), len(BIN_LABELS)), dtype=int)
+    for actual, pred in zip(actual_bins, pred_bins):
+        confusion[actual, pred] += 1
+
+    exact = float(np.mean(actual_bins == pred_bins))
+    adjacent = float(np.mean(np.abs(actual_bins - pred_bins) <= 1))
+    lines = [
+        f"Bin summary: exact={exact:.3f}  within_adjacent={adjacent:.3f}",
+        "  actual/pred      " + "  ".join(f"{label:>8s}" for label in BIN_LABELS),
+    ]
+    for idx, label in enumerate(BIN_LABELS):
+        row = confusion[idx]
+        n = int(row.sum())
+        row_text = "  ".join(f"{v:8d}" for v in row)
+        lines.append(f"  {label:>11s} n={n:3d}  {row_text}")
+
+    lines.append("  per-actual-bin:")
+    for idx, label in enumerate(BIN_LABELS):
+        mask = actual_bins == idx
+        if not mask.any():
+            continue
+        stats = _metrics(y_true[valid][mask], y_pred[valid][mask])
+        lines.append(
+            f"    {label:>7s} n={stats['n']:.0f} "
+            f"mean_pred={float(np.mean(y_pred[valid][mask])):.2f} "
+            f"bias={stats['bias']:.2f} mae={stats['mae']:.2f} rmse={stats['rmse']:.2f} "
+            f"within_3={stats['within_3']:.3f} within_5={stats['within_5']:.3f}"
+        )
+    return lines
+
+
+def _plot_one(
+    y_true: np.ndarray,
+    y_pred: np.ndarray,
+    folds: np.ndarray,
+    *,
+    title: str,
+    output: Path,
+    dpi: int,
+) -> None:
+    stats = _metrics(y_true, y_pred)
+    residual = y_pred - y_true
+
+    lo = float(np.nanmin([y_true.min(), y_pred.min()]))
+    hi = float(np.nanmax([y_true.max(), y_pred.max()]))
+    pad = max((hi - lo) * 0.05, 1.0)
+    lo -= pad
+    hi += pad
+
+    fig, (ax_scatter, ax_resid) = plt.subplots(
+        1,
+        2,
+        figsize=(11.5, 5.0),
+        gridspec_kw={"width_ratios": [1.4, 1.0]},
+        constrained_layout=True,
+    )
+
+    scatter = ax_scatter.scatter(
+        y_true,
+        y_pred,
+        c=folds,
+        cmap="tab10",
+        s=34,
+        alpha=0.72,
+        linewidths=0,
+    )
+    ax_scatter.plot([lo, hi], [lo, hi], color="black", linewidth=1.2, linestyle="--", label="ideal")
+    if y_true.size > 1:
+        slope, intercept = np.polyfit(y_true, y_pred, deg=1)
+        ax_scatter.plot(
+            [lo, hi],
+            [slope * lo + intercept, slope * hi + intercept],
+            color="#b03a2e",
+            linewidth=1.4,
+            label=f"fit: y={slope:.2f}x{intercept:+.2f}",
+        )
+    ax_scatter.set_xlim(lo, hi)
+    ax_scatter.set_ylim(lo, hi)
+    ax_scatter.set_aspect("equal", adjustable="box")
+    ax_scatter.set_xlabel("Actual")
+    ax_scatter.set_ylabel("Predicted")
+    ax_scatter.set_title(title)
+    ax_scatter.grid(True, color="#e6e6e6", linewidth=0.8)
+    ax_scatter.legend(loc="upper left", frameon=False)
+
+    cbar = fig.colorbar(scatter, ax=ax_scatter, fraction=0.046, pad=0.04)
+    cbar.set_label("Fold")
+
+    text = (
+        f"n={stats['n']:.0f}\n"
+        f"MSE={stats['mse']:.3f}\n"
+        f"RMSE={stats['rmse']:.3f}\n"
+        f"MAE={stats['mae']:.3f}\n"
+        f"bias={stats['bias']:.3f}\n"
+        f"r={stats['r']:.3f}\n"
+        f"±1={stats['within_1']:.3f}\n"
+        f"±3={stats['within_3']:.3f}\n"
+        f"±5={stats['within_5']:.3f}"
+    )
+    ax_scatter.text(
+        0.98,
+        0.02,
+        text,
+        transform=ax_scatter.transAxes,
+        ha="right",
+        va="bottom",
+        fontsize=9,
+        bbox={"boxstyle": "round,pad=0.35", "facecolor": "white", "edgecolor": "#cccccc", "alpha": 0.92},
+    )
+
+    bins = min(30, max(8, int(np.sqrt(residual.size))))
+    ax_resid.hist(residual, bins=bins, color="#4c78a8", alpha=0.85, edgecolor="white")
+    ax_resid.axvline(0, color="black", linewidth=1.1, linestyle="--")
+    ax_resid.axvline(stats["bias"], color="#b03a2e", linewidth=1.4, label="mean residual")
+    ax_resid.set_xlabel("Predicted - actual")
+    ax_resid.set_ylabel("Count")
+    ax_resid.set_title("Residuals")
+    ax_resid.grid(True, axis="y", color="#e6e6e6", linewidth=0.8)
+    ax_resid.legend(loc="upper right", frameon=False)
+
+    output.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(output, dpi=dpi)
+    plt.close(fig)
+
+
+def default_output_path(path: Path, phase: str, head: str, split: str, label: str) -> Path:
+    stem = f"regression_predictions_{phase}_{head}_{split}"
+    if label:
+        stem += f"_{label}"
+    stem += ".png"
+    return path.with_name(stem)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("path", type=Path, help="A predictions.h5 file or a directory containing predictions.h5 files")
+    ap.add_argument("--phase", default=None, help="Phase group to plot; defaults to the first logits phase")
+    ap.add_argument("--head", default=None, help="Head name to plot; defaults to the first head")
+    ap.add_argument("--split", default="test", help="Split label to plot: test, val, train, or all")
+    ap.add_argument("--epoch", type=int, default=-1, help="Epoch index to plot; negative values count from the end")
+    ap.add_argument("--output-index", type=int, default=0, help="Regression output index within logits' final axis")
+    ap.add_argument("--glaucoma-only", action="store_true", help="Only plot rows where actual y_true is non-zero")
+    ap.add_argument("--out-dir", type=Path, default=None, help="Optional directory for all output PNGs")
+    ap.add_argument("--bin-summary", action="store_true", help="Print severity-bin accuracy and confusion matrix")
+    ap.add_argument("--dpi", type=int, default=160, help="Output PNG resolution")
+    args = ap.parse_args()
+
+    filter_label = "glaucoma_only" if args.glaucoma_only else ""
+    for pred_path in _find_prediction_files(args.path):
+        y_true, y_pred, folds, phase, head = _collect_points(
+            pred_path,
+            phase=args.phase,
+            head=args.head,
+            split=args.split,
+            epoch=args.epoch,
+            output_index=args.output_index,
+            glaucoma_only=args.glaucoma_only,
+        )
+        rel_title = pred_path.parent.as_posix()
+        filter_text = ", glaucoma only" if args.glaucoma_only else ""
+        title = f"{rel_title}\nphase={phase}, head={head}, split={args.split}, epoch={args.epoch}{filter_text}"
+        if args.out_dir is None:
+            out_path = default_output_path(pred_path, phase, head, args.split, filter_label)
+        else:
+            rep_name = "_".join(pred_path.parent.parts[-3:])
+            suffix = f"_{filter_label}" if filter_label else ""
+            out_path = args.out_dir / f"{rep_name}_{phase}_{head}_{args.split}{suffix}.png"
+        _plot_one(y_true, y_pred, folds, title=title, output=out_path, dpi=args.dpi)
+        stats = _metrics(y_true, y_pred)
+        print(
+            f"{out_path}  n={stats['n']:.0f}  "
+            f"mse={stats['mse']:.4f}  rmse={stats['rmse']:.4f}  "
+            f"mae={stats['mae']:.4f}  r={stats['r']:.4f}  "
+            f"within_1={stats['within_1']:.3f}  "
+            f"within_3={stats['within_3']:.3f}  "
+            f"within_5={stats['within_5']:.3f}"
+        )
+        if args.bin_summary:
+            print("\n".join(_bin_summary_lines(y_true, y_pred)))
+
+
+if __name__ == "__main__":
+    main()

v4/scripts/analysis/severity_confusion.py

@@ -0,0 +1,188 @@
+"""3-bin severity confusion matrix for vf_md regression heads.
+
+Reads predictions.h5 files from a regression run, bins both actual and predicted
+MD into 3 severity tiers (severe / moderate / no-problem), and reports:
+
+  * confusion matrix (counts and per-row %)
+  * exact-bin & adjacent-bin accuracy
+  * per-bin recall
+  * binary "no-problem vs disease" sensitivity/specificity at the −4.5 dB boundary
+  * optional saved heatmap PNG
+
+Bin boundaries (placed halfway between integer bins, matching plot_regression_predictions):
+  severe       :  vf_md <= -9.5
+  moderate     : -9.5 < vf_md <= -4.5
+  no-problem   : vf_md > -4.5
+
+Usage:
+    python -m v4.scripts.analysis.severity_confusion \\
+        v4/results/experiments/reg_head/baseline_reg_nt50 \\
+        --save-fig analysis/figures/regression_severity_confusion.png
+"""
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+import h5py
+import numpy as np
+
+LABELS = ["severe (≤−10)", "moderate (−9..−5)", "no-problem (≥−4)"]
+
+
+def severity_3bin(values: np.ndarray) -> np.ndarray:
+    bins = np.full(values.shape, -1, dtype=int)
+    bins[values <= -9.5] = 0
+    bins[(values > -9.5) & (values <= -4.5)] = 1
+    bins[values > -4.5] = 2
+    return bins
+
+
+def _decode(arr) -> np.ndarray:
+    return np.array([s.decode("utf-8") if isinstance(s, bytes) else str(s) for s in arr])
+
+
+def collect_test_predictions(path: Path) -> tuple[np.ndarray, np.ndarray] | None:
+    with h5py.File(path, "r") as f:
+        if "hb" not in f:
+            return None
+        grp = f["hb"]
+        logits = grp["logits"][:]
+        y_true = grp["y_true"][:].astype(float)
+        split  = grp["split"][:]
+        n_folds, n_epochs, _, n_heads, _ = logits.shape
+        ep, head, out = n_epochs - 1, n_heads - 1, 0
+        actuals: list[np.ndarray] = []
+        preds:   list[np.ndarray] = []
+        for fold in range(n_folds):
+            labels = _decode(split[fold])
+            mask = (labels == "test") & np.isfinite(y_true) & np.isfinite(logits[fold, ep, :, head, out])
+            if not mask.any():
+                continue
+            actuals.append(y_true[mask])
+            preds.append(logits[fold, ep, mask, head, out].astype(float))
+    if not actuals:
+        return None
+    return np.concatenate(actuals), np.concatenate(preds)
+
+
+def report(actuals: np.ndarray, preds: np.ndarray) -> dict:
+    ab = severity_3bin(actuals)
+    pb = severity_3bin(preds)
+    valid = (ab >= 0) & (pb >= 0)
+    ab, pb = ab[valid], pb[valid]
+    cm = np.zeros((3, 3), dtype=int)
+    for x, y in zip(ab, pb):
+        cm[x, y] += 1
+
+    # Binary disease vs no-problem (bins 0+1 vs bin 2)
+    actual_disease = ab <= 1
+    pred_disease   = pb <= 1
+    tp = int(np.sum(actual_disease & pred_disease))
+    tn = int(np.sum(~actual_disease & ~pred_disease))
+    fp = int(np.sum(~actual_disease & pred_disease))
+    fn = int(np.sum(actual_disease & ~pred_disease))
+    sens = tp / max(tp + fn, 1)
+    spec = tn / max(tn + fp, 1)
+
+    return {
+        "confusion":     cm,
+        "n_test":        int(ab.size),
+        "exact_acc":     float(np.mean(ab == pb)),
+        "adjacent_acc":  float(np.mean(np.abs(ab - pb) <= 1)),
+        "recall_per_bin": [float(np.mean(pb[ab == i] == i)) if (ab == i).any() else float("nan")
+                            for i in range(3)],
+        "n_per_bin":     [int((ab == i).sum()) for i in range(3)],
+        "binary_sens":   sens,
+        "binary_spec":   spec,
+        "binary_tp":     tp,
+        "binary_fp":     fp,
+        "binary_fn":     fn,
+        "binary_tn":     tn,
+    }
+
+
+def print_report(r: dict) -> None:
+    cm = r["confusion"]
+    print(f"\nn_test (pooled across reps × folds): {r['n_test']}")
+    print(f"\nConfusion matrix (rows = actual, cols = predicted):")
+    print(f"{'actual \\ pred':<22s}  {LABELS[0]:>16s}  {LABELS[1]:>20s}  {LABELS[2]:>18s}    n")
+    for i in range(3):
+        row = cm[i]
+        print(f"{LABELS[i]:<22s}  {row[0]:>16d}  {row[1]:>20d}  {row[2]:>18d}    {row.sum()}")
+    print(f"\nExact-bin accuracy:    {r['exact_acc']:.3f}")
+    print(f"Adjacent-bin accuracy: {r['adjacent_acc']:.3f}")
+    print("\nPer-bin recall:")
+    for i in range(3):
+        n = r["n_per_bin"][i]
+        rec = r["recall_per_bin"][i]
+        print(f"  {LABELS[i]:<22s}  n={n:>4d}  recall={rec:.3f}")
+    print(f"\nBinary disease (severe+moderate) vs no-problem, threshold = −4.5 dB:")
+    print(f"  sensitivity (correctly flag disease):   {r['binary_sens']:.3f}  ({r['binary_tp']}/{r['binary_tp']+r['binary_fn']})")
+    print(f"  specificity (correctly clear healthy):  {r['binary_spec']:.3f}  ({r['binary_tn']}/{r['binary_tn']+r['binary_fp']})")
+
+
+def save_heatmap(r: dict, path: Path) -> None:
+    import matplotlib.pyplot as plt
+    cm = r["confusion"]
+    cm_pct = cm / np.maximum(cm.sum(axis=1, keepdims=True), 1)
+
+    fig, ax = plt.subplots(figsize=(6.5, 5.5))
+    im = ax.imshow(cm_pct, cmap="Blues", vmin=0, vmax=1, aspect="equal")
+    for i in range(3):
+        for j in range(3):
+            ax.text(j, i, f"{cm[i,j]}\n({cm_pct[i,j]*100:.1f}%)",
+                    ha="center", va="center",
+                    color="white" if cm_pct[i,j] > 0.5 else "black",
+                    fontsize=10)
+    ax.set_xticks(range(3)); ax.set_xticklabels(LABELS, rotation=20, ha="right")
+    ax.set_yticks(range(3)); ax.set_yticklabels(LABELS)
+    ax.set_xlabel("Predicted")
+    ax.set_ylabel("Actual")
+    ax.set_title(f"VF-MD severity confusion (n={r['n_test']})\n"
+                 f"exact={r['exact_acc']:.3f}  adjacent={r['adjacent_acc']:.3f}  "
+                 f"sens={r['binary_sens']:.3f}  spec={r['binary_spec']:.3f}")
+    fig.colorbar(im, ax=ax, label="Row-normalised fraction")
+    fig.tight_layout()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"\nSaved heatmap: {path}")
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("path", type=Path, help="A predictions.h5 file or a directory containing them")
+    ap.add_argument("--save-fig", type=Path, default=None, help="Optional path to save the confusion-matrix heatmap PNG")
+    args = ap.parse_args()
+
+    if args.path.is_file():
+        files = [args.path]
+    else:
+        files = sorted(args.path.rglob("predictions.h5"))
+    if not files:
+        raise SystemExit(f"No predictions.h5 under {args.path}")
+
+    all_actual: list[np.ndarray] = []
+    all_pred:   list[np.ndarray] = []
+    for fp in files:
+        res = collect_test_predictions(fp)
+        if res is None:
+            print(f"  skipped (no hb predictions): {fp}")
+            continue
+        a, p = res
+        all_actual.append(a); all_pred.append(p)
+    if not all_actual:
+        raise SystemExit("No usable predictions found")
+
+    actuals = np.concatenate(all_actual)
+    preds   = np.concatenate(all_pred)
+    print(f"Pooled across {len(all_actual)} predictions.h5 files")
+    r = report(actuals, preds)
+    print_report(r)
+    if args.save_fig:
+        save_heatmap(r, args.save_fig)
+
+
+if __name__ == "__main__":
+    main()

v4/scripts/analysis/summarize_run.py

@@ -39,14 +39,20 @@ def find_rep_summaries(run_dir: Path) -> list[tuple[int, Path]]:
 
 
 def load_rep(summary_path: Path) -> dict:
-    """Extract the fields we summarise from one rep's summary.json."""
+    """Extract the fields we summarise from one rep's summary.json.
+
+    Uses `primary_metric` if present (for regression runs e.g. neg_mse),
+    falling back to AUC for legacy classification runs.
+    """
     d = json.loads(summary_path.read_text())
+    pm = d.get("primary_metric", "auc")
     return {
-        "val_mean":  float(d.get("mean_val_auc",  float("nan"))),
-        "val_std":   float(d.get("std_val_auc",   float("nan"))),
-        "test_mean": float(d.get("mean_test_auc", float("nan"))),
-        "test_std":  float(d.get("std_test_auc",  float("nan"))),
-        "elapsed_s": float(d.get("elapsed_s",     float("nan"))),
+        "primary":    pm,
+        "val_mean":   float(d.get(f"mean_val_{pm}",  d.get("mean_val_auc",  float("nan")))),
+        "val_std":    float(d.get(f"std_val_{pm}",   d.get("std_val_auc",   float("nan")))),
+        "test_mean":  float(d.get(f"mean_test_{pm}", d.get("mean_test_auc", float("nan")))),
+        "test_std":   float(d.get(f"std_test_{pm}",  d.get("std_test_auc",  float("nan")))),
+        "elapsed_s":  float(d.get("elapsed_s",       float("nan"))),
         "eval_stage": d.get("eval_stage", "?"),
     }
 
@@ -63,6 +69,7 @@ def summarise(run_dir: Path) -> dict:
         "run":         str(run_dir),
         "n_reps":      len(rows),
         "eval_stage":  rows[0][1]["eval_stage"],
+        "primary":     rows[0][1]["primary"],
         "val_mean":    float(np.mean(val)),
         "val_std":     float(np.std(val)),
         "val_min":     float(np.min(val)),
@@ -83,12 +90,13 @@ def render(s: dict, per_rep: bool = False) -> str:
     if s["n_reps"] == 0:
         return f"Run: {s['run']}\n  no reps with summary.json found."
 
+    metric = s.get("primary", "auc")
     lines = [
         f"Run:        {s['run']}",
-        f"Reps:       {s['n_reps']}  (eval_stage={s['eval_stage']})",
-        f"Val  AUC:   {s['val_mean']:.4f} ± {s['val_std']:.4f}   "
+        f"Reps:       {s['n_reps']}  (eval_stage={s['eval_stage']}, metric={metric})",
+        f"Val  {metric}:   {s['val_mean']:.4f} ± {s['val_std']:.4f}   "
         f"[min={s['val_min']:.4f}  max={s['val_max']:.4f}]",
-        f"Test AUC:   {s['test_mean']:.4f} ± {s['test_std']:.4f}   "
+        f"Test {metric}:   {s['test_mean']:.4f} ± {s['test_std']:.4f}   "
         f"[min={s['test_min']:.4f}  max={s['test_max']:.4f}]",
     ]
     if s.get("elapsed_total_s") is not None: