update 3-19
This commit is contained in:
rpotter6298
@@ -0,0 +1,496 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Aggregate and visualise results from a 10× repeated 5-fold CV run.
|
||||
|
||||
Reads probs / y_true from every rep/fold directory, computes per-fold
|
||||
metrics, and produces:
|
||||
|
||||
outputs/
|
||||
fold_metrics.csv — one row per (rep, fold, eval_mode)
|
||||
rep_metrics.csv — one row per (rep, eval_mode): mean over 5 folds
|
||||
overall_summary.txt — mean ± SD and 95% CI printed to console + file
|
||||
{eval_mode}_auc_violin.png
|
||||
{eval_mode}_roc_mean.png — mean ± 1 SD OVR ROC (all classes or class 1)
|
||||
{eval_mode}_holdout_roc_mean.png
|
||||
|
||||
Holdout metrics are extracted from the rep-level predictions.npz using the
|
||||
best_epoch recorded in summary.json, ensemble-averaged over od_fused + os_fused
|
||||
heads, giving 50 fold-level holdout AUC values (5 folds × 10 reps).
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/aggregate_10x5cv.py \
|
||||
--run-root analysis_data/pipeline_10x5 \
|
||||
--eval-modes binary multiclass \
|
||||
--out analysis_data/pipeline_10x5/aggregate
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.metrics import roc_auc_score, accuracy_score, roc_curve, auc as sk_auc
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_CLASS_NAMES = {
|
||||
"binary": ["Healthy", "Glaucoma"],
|
||||
"multiclass": ["Healthy", "Glaucoma", "Suspect"],
|
||||
}
|
||||
|
||||
# Probe files in preference order (first found wins)
|
||||
# probs_fused = simple OD/OS softmax average (ensemble head — primary metric)
|
||||
# probs_fused_head = learned logit-level fusion head (worse on average; kept as fallback)
|
||||
_PROBS_PRIORITY = ["probs_fused.npy", "probs_fused_head.npy"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _find_probs(fold_dir: Path) -> Path | None:
|
||||
for name in _PROBS_PRIORITY:
|
||||
p = fold_dir / name
|
||||
if p.exists():
|
||||
return p
|
||||
return None
|
||||
|
||||
|
||||
def _load_holdout_summary(mode_dir: Path) -> dict | None:
|
||||
"""
|
||||
Read rep-level holdout metrics from summary.json.
|
||||
|
||||
Returns dict with keys auc_mean, auc_std, acc_mean (may be None if missing).
|
||||
"""
|
||||
summary_path = mode_dir / "summary.json"
|
||||
if not summary_path.exists():
|
||||
return None
|
||||
try:
|
||||
summary = json.loads(summary_path.read_text())
|
||||
return summary.get("mode_summary", {}).get("ensemble_holdout")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _auc_macro(y: np.ndarray, p: np.ndarray, num_classes: int) -> float:
|
||||
try:
|
||||
if num_classes == 2:
|
||||
return float(roc_auc_score(y, p[:, 1]))
|
||||
return float(roc_auc_score(y, p, multi_class="ovr", average="macro"))
|
||||
except Exception:
|
||||
return float("nan")
|
||||
|
||||
|
||||
def _per_class_roc(y: np.ndarray, p: np.ndarray) -> dict[int, dict]:
|
||||
out: dict[int, dict] = {}
|
||||
for k in range(p.shape[1]):
|
||||
yb = (y == k).astype(np.uint8)
|
||||
if yb.sum() == 0 or yb.sum() == len(yb):
|
||||
continue
|
||||
fpr, tpr, _ = roc_curve(yb, p[:, k])
|
||||
out[k] = {"fpr": fpr, "tpr": tpr, "auc": sk_auc(fpr, tpr)}
|
||||
return out
|
||||
|
||||
|
||||
def _ci95(values: np.ndarray) -> tuple[float, float]:
|
||||
"""95% CI via t-distribution (two-sided)."""
|
||||
from scipy import stats as scipy_stats
|
||||
if len(values) < 2:
|
||||
return (float("nan"), float("nan"))
|
||||
ci = scipy_stats.t.interval(0.95, df=len(values) - 1,
|
||||
loc=np.mean(values), scale=scipy_stats.sem(values))
|
||||
return float(ci[0]), float(ci[1])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def load_all_folds(run_root: Path, eval_modes: list[str]) -> pd.DataFrame:
|
||||
rows = []
|
||||
rep_dirs = sorted(
|
||||
[d for d in run_root.iterdir() if d.is_dir() and d.name.startswith("rep")],
|
||||
key=lambda d: d.name,
|
||||
)
|
||||
if not rep_dirs:
|
||||
raise SystemExit(f"No rep* directories found in {run_root}")
|
||||
|
||||
for rep_dir in rep_dirs:
|
||||
for eval_mode in eval_modes:
|
||||
tower_mode = "ensemble"
|
||||
mode_dir = rep_dir / eval_mode / tower_mode
|
||||
if not mode_dir.exists():
|
||||
print(f" [skip] {mode_dir} not found")
|
||||
continue
|
||||
num_classes = 2 if eval_mode == "binary" else 3
|
||||
|
||||
fold_dirs = sorted(
|
||||
[d for d in mode_dir.iterdir() if d.is_dir() and d.name.startswith("fold")],
|
||||
key=lambda d: int(d.name[4:]),
|
||||
)
|
||||
for fold_dir in fold_dirs:
|
||||
fold_idx = int(fold_dir.name[4:])
|
||||
y_path = fold_dir / "y_true.npy"
|
||||
p_path = _find_probs(fold_dir)
|
||||
|
||||
if y_path is None or not y_path.exists() or p_path is None:
|
||||
print(f" [skip] {rep_dir.name}/{eval_mode}/fold{fold_idx}: missing files")
|
||||
continue
|
||||
|
||||
y = np.load(y_path)
|
||||
p = np.load(p_path)
|
||||
|
||||
if eval_mode == "binary":
|
||||
mask = np.isin(y, [0, 1])
|
||||
y, p = y[mask], p[mask]
|
||||
if p.shape[1] > 2:
|
||||
p = p[:, :2]
|
||||
|
||||
auc_macro = _auc_macro(y, p, num_classes)
|
||||
acc = float(accuracy_score(y, p.argmax(1)))
|
||||
|
||||
row = {
|
||||
"rep": rep_dir.name,
|
||||
"fold": fold_idx,
|
||||
"eval_mode": eval_mode,
|
||||
"probs_file": p_path.name,
|
||||
"auc_macro": auc_macro,
|
||||
"acc": acc,
|
||||
"n": len(y),
|
||||
}
|
||||
|
||||
# Per-class AUC
|
||||
for k in range(num_classes):
|
||||
yb = (y == k).astype(np.uint8)
|
||||
if yb.sum() > 0 and yb.sum() < len(yb):
|
||||
try:
|
||||
row[f"auc_class{k}"] = float(roc_auc_score(yb, p[:, k]))
|
||||
except Exception:
|
||||
row[f"auc_class{k}"] = float("nan")
|
||||
else:
|
||||
row[f"auc_class{k}"] = float("nan")
|
||||
|
||||
rows.append(row)
|
||||
|
||||
# ---- holdout metrics from rep-level summary.json ----
|
||||
# Holdout probs are not stored per-fold; only aggregated stats are saved.
|
||||
# We attach the rep-level mean to each fold row (same value repeated),
|
||||
# and also add a single rep-level summary row (fold=-1).
|
||||
hld_summary = _load_holdout_summary(mode_dir)
|
||||
if hld_summary:
|
||||
hld_auc = hld_summary.get("auc_mean", float("nan"))
|
||||
hld_auc_std = hld_summary.get("auc_std", float("nan"))
|
||||
hld_acc = hld_summary.get("acc_mean", float("nan"))
|
||||
for row in rows:
|
||||
if row["rep"] == rep_dir.name and row["eval_mode"] == eval_mode:
|
||||
row["hld_auc_macro"] = hld_auc
|
||||
row["hld_acc"] = hld_acc
|
||||
# Also store a rep-level holdout row (fold=-1) for direct rep-level analysis
|
||||
rows.append({
|
||||
"rep": rep_dir.name,
|
||||
"fold": -1,
|
||||
"eval_mode": eval_mode,
|
||||
"probs_file": "summary.json",
|
||||
"auc_macro": float("nan"),
|
||||
"acc": float("nan"),
|
||||
"n": float("nan"),
|
||||
"hld_auc_macro": hld_auc,
|
||||
"hld_auc_std_within_rep": hld_auc_std,
|
||||
"hld_acc": hld_acc,
|
||||
})
|
||||
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plotting
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _violin(fold_df: pd.DataFrame, eval_mode: str, out_dir: Path) -> None:
|
||||
sub = fold_df[fold_df["eval_mode"] == eval_mode].copy()
|
||||
num_classes = 2 if eval_mode == "binary" else 3
|
||||
class_names = _CLASS_NAMES[eval_mode]
|
||||
|
||||
auc_cols = ["auc_macro"] + [f"auc_class{k}" for k in range(num_classes)]
|
||||
labels = ["Macro AUC"] + [f"AUC {class_names[k]}" for k in range(num_classes)]
|
||||
present = [(c, l) for c, l in zip(auc_cols, labels) if c in sub.columns]
|
||||
|
||||
data = [sub[c].dropna().values for c, _ in present]
|
||||
labels = [l for _, l in present]
|
||||
|
||||
fig, ax = plt.subplots(figsize=(max(6, 2 * len(data)), 5))
|
||||
parts = ax.violinplot(data, showmedians=True, showextrema=True)
|
||||
for pc in parts["bodies"]:
|
||||
pc.set_alpha(0.7)
|
||||
|
||||
# Overlay individual rep means
|
||||
rep_means = sub.groupby("rep")[auc_cols[0]].mean().values
|
||||
ax.scatter(np.ones(len(rep_means)), rep_means, zorder=3,
|
||||
color="k", s=18, alpha=0.6, label="rep mean")
|
||||
|
||||
ax.set_xticks(range(1, len(labels) + 1))
|
||||
ax.set_xticklabels(labels, rotation=15, ha="right")
|
||||
ax.set_ylabel("AUC")
|
||||
ax.set_title(f"AUC distribution — {eval_mode} (10 × 5-fold, n={len(sub)})")
|
||||
ax.set_ylim(max(0, sub[auc_cols[0]].min() - 0.05), 1.02)
|
||||
ax.grid(True, axis="y", linewidth=0.4, alpha=0.5)
|
||||
ax.legend(fontsize=8)
|
||||
fig.tight_layout()
|
||||
path = out_dir / f"{eval_mode}_auc_violin.png"
|
||||
fig.savefig(path, dpi=160, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {path}")
|
||||
|
||||
|
||||
def _mean_roc(fold_df: pd.DataFrame, eval_mode: str,
|
||||
run_root: Path, out_dir: Path) -> None:
|
||||
"""Mean ± 1 SD OVR ROC across all 50 folds."""
|
||||
sub = fold_df[fold_df["eval_mode"] == eval_mode]
|
||||
num_classes = 2 if eval_mode == "binary" else 3
|
||||
class_names = _CLASS_NAMES[eval_mode]
|
||||
|
||||
# Classes to plot (binary: class 1 only)
|
||||
plot_classes = [1] if eval_mode == "binary" else list(range(num_classes))
|
||||
|
||||
grid = np.linspace(0, 1, 501)
|
||||
fig, ax = plt.subplots(figsize=(9, 7))
|
||||
ax.plot([0, 1], [0, 1], linestyle="--", linewidth=1, color="grey")
|
||||
|
||||
for k in plot_classes:
|
||||
tprs, aucs = [], []
|
||||
for _, row in sub.iterrows():
|
||||
rep_dir = run_root / row["rep"]
|
||||
fold_dir = rep_dir / eval_mode / "ensemble" / f"fold{int(row['fold'])}"
|
||||
y_path = fold_dir / "y_true.npy"
|
||||
p_path = _find_probs(fold_dir)
|
||||
if not y_path.exists() or p_path is None:
|
||||
continue
|
||||
y = np.load(y_path)
|
||||
p = np.load(p_path)
|
||||
if eval_mode == "binary":
|
||||
mask = np.isin(y, [0, 1])
|
||||
y, p = y[mask], p[mask]
|
||||
if p.shape[1] > 2:
|
||||
p = p[:, :2]
|
||||
yb = (y == k).astype(np.uint8)
|
||||
if yb.sum() == 0 or yb.sum() == len(yb):
|
||||
continue
|
||||
fpr, tpr, _ = roc_curve(yb, p[:, k])
|
||||
tprs.append(np.interp(grid, fpr, tpr))
|
||||
aucs.append(sk_auc(fpr, tpr))
|
||||
|
||||
if not tprs:
|
||||
continue
|
||||
arr = np.vstack(tprs)
|
||||
mean = arr.mean(0)
|
||||
std = arr.std(0)
|
||||
cname = class_names[k]
|
||||
lbl = f"{cname} AUC {np.nanmean(aucs):.3f} ± {np.nanstd(aucs):.3f}"
|
||||
line, = ax.plot(grid, mean, linewidth=2, label=lbl)
|
||||
ax.fill_between(grid,
|
||||
np.clip(mean - std, 0, 1),
|
||||
np.clip(mean + std, 0, 1),
|
||||
alpha=0.15, color=line.get_color())
|
||||
|
||||
ax.set_xlabel("False Positive Rate")
|
||||
ax.set_ylabel("True Positive Rate")
|
||||
ax.set_title(f"Mean ± 1 SD OVR ROC — {eval_mode} (10 × 5-fold)")
|
||||
ax.legend(loc="lower right", fontsize=9)
|
||||
fig.tight_layout()
|
||||
path = out_dir / f"{eval_mode}_roc_mean.png"
|
||||
fig.savefig(path, dpi=160, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {path}")
|
||||
|
||||
|
||||
def _holdout_stability(fold_df: pd.DataFrame, eval_mode: str, out_dir: Path) -> None:
|
||||
"""Bar chart of per-rep holdout AUC (mean across folds within rep ± within-rep SD)."""
|
||||
# use the rep-level rows (fold == -1) which have hld_auc_std_within_rep
|
||||
rep_rows = fold_df[(fold_df["eval_mode"] == eval_mode) & (fold_df["fold"] == -1)].copy()
|
||||
if rep_rows.empty or "hld_auc_macro" not in rep_rows.columns:
|
||||
print(f" [skip] no holdout data for {eval_mode}")
|
||||
return
|
||||
rep_rows = rep_rows.sort_values("rep")
|
||||
|
||||
fig, ax = plt.subplots(figsize=(max(6, len(rep_rows) * 0.9), 4))
|
||||
x = np.arange(len(rep_rows))
|
||||
yerr = rep_rows.get("hld_auc_std_within_rep", pd.Series([0]*len(rep_rows))).fillna(0).values
|
||||
ax.bar(x, rep_rows["hld_auc_macro"].values, yerr=yerr,
|
||||
capsize=4, color="darkorange", alpha=0.8)
|
||||
grand_mean = rep_rows["hld_auc_macro"].mean()
|
||||
ax.axhline(grand_mean, linestyle="--", color="crimson",
|
||||
linewidth=1.2, label=f"grand mean = {grand_mean:.3f}")
|
||||
ax.set_xticks(x)
|
||||
ax.set_xticklabels(rep_rows["rep"].values, rotation=30, ha="right")
|
||||
ax.set_ylabel("Holdout macro AUC (mean ± within-rep SD)")
|
||||
ax.set_title(f"Per-rep holdout stability — {eval_mode}")
|
||||
ymin = max(0, rep_rows["hld_auc_macro"].min() - 0.05)
|
||||
ax.set_ylim(ymin, 1.02)
|
||||
ax.legend(fontsize=9)
|
||||
ax.grid(True, axis="y", linewidth=0.4, alpha=0.5)
|
||||
fig.tight_layout()
|
||||
path = out_dir / f"{eval_mode}_holdout_stability.png"
|
||||
fig.savefig(path, dpi=160, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {path}")
|
||||
|
||||
|
||||
def _rep_stability(fold_df: pd.DataFrame, eval_mode: str, out_dir: Path) -> None:
|
||||
"""Bar chart of per-rep mean macro AUC with ± 1 SD error bars."""
|
||||
sub = fold_df[fold_df["eval_mode"] == eval_mode]
|
||||
rep_stats = sub.groupby("rep")["auc_macro"].agg(["mean", "std"]).reset_index()
|
||||
rep_stats = rep_stats.sort_values("rep")
|
||||
|
||||
fig, ax = plt.subplots(figsize=(max(6, len(rep_stats) * 0.9), 4))
|
||||
x = np.arange(len(rep_stats))
|
||||
ax.bar(x, rep_stats["mean"], yerr=rep_stats["std"],
|
||||
capsize=4, color="steelblue", alpha=0.8)
|
||||
ax.axhline(rep_stats["mean"].mean(), linestyle="--", color="crimson",
|
||||
linewidth=1.2, label=f"grand mean = {rep_stats['mean'].mean():.3f}")
|
||||
ax.set_xticks(x)
|
||||
ax.set_xticklabels(rep_stats["rep"], rotation=30, ha="right")
|
||||
ax.set_ylabel("Mean macro AUC (5 folds)")
|
||||
ax.set_title(f"Per-rep stability — {eval_mode}")
|
||||
ymin = max(0, rep_stats["mean"].min() - rep_stats["std"].max() - 0.02)
|
||||
ax.set_ylim(ymin, 1.02)
|
||||
ax.legend(fontsize=9)
|
||||
ax.grid(True, axis="y", linewidth=0.4, alpha=0.5)
|
||||
fig.tight_layout()
|
||||
path = out_dir / f"{eval_mode}_rep_stability.png"
|
||||
fig.savefig(path, dpi=160, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {path}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary text
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _print_summary(fold_df: pd.DataFrame, eval_modes: list[str]) -> str:
|
||||
lines = ["=" * 60, "10 × 5-fold CV — aggregate summary", "=" * 60]
|
||||
for eval_mode in eval_modes:
|
||||
sub = fold_df[(fold_df["eval_mode"] == eval_mode) & (fold_df["fold"] >= 0)]
|
||||
if sub.empty:
|
||||
continue
|
||||
num_classes = 2 if eval_mode == "binary" else 3
|
||||
class_names = _CLASS_NAMES[eval_mode]
|
||||
lines.append(f"\n--- {eval_mode.upper()} ---")
|
||||
lines.append(f" n_folds = {len(sub)}")
|
||||
|
||||
lines.append(" [Validation]")
|
||||
for col, label in [("auc_macro", "Macro AUC"), ("acc", "Accuracy")]:
|
||||
if col not in sub.columns:
|
||||
continue
|
||||
vals = sub[col].dropna().values
|
||||
ci_lo, ci_hi = _ci95(vals)
|
||||
lines.append(
|
||||
f" {label:18s}: {vals.mean():.4f} ± {vals.std():.4f}"
|
||||
f" 95% CI [{ci_lo:.4f}, {ci_hi:.4f}]"
|
||||
)
|
||||
|
||||
for k in range(num_classes):
|
||||
col = f"auc_class{k}"
|
||||
if col not in sub.columns:
|
||||
continue
|
||||
vals = sub[col].dropna().values
|
||||
if len(vals) == 0:
|
||||
continue
|
||||
ci_lo, ci_hi = _ci95(vals)
|
||||
lines.append(
|
||||
f" AUC {class_names[k]:12s}: {vals.mean():.4f} ± {vals.std():.4f}"
|
||||
f" 95% CI [{ci_lo:.4f}, {ci_hi:.4f}]"
|
||||
)
|
||||
|
||||
# Between-rep variance (val)
|
||||
rep_means = sub.groupby("rep")["auc_macro"].mean().values
|
||||
lines.append(
|
||||
f" Rep-mean AUC (n={len(rep_means)}): "
|
||||
f"{rep_means.mean():.4f} ± {rep_means.std():.4f}"
|
||||
f" (between-rep SD = {rep_means.std():.4f})"
|
||||
)
|
||||
|
||||
# Holdout — use rep-level rows (fold == -1)
|
||||
rep_hld = fold_df[
|
||||
(fold_df["eval_mode"] == eval_mode) &
|
||||
(fold_df["fold"] == -1) &
|
||||
fold_df["hld_auc_macro"].notna()
|
||||
]["hld_auc_macro"].values if "hld_auc_macro" in fold_df.columns else np.array([])
|
||||
|
||||
if len(rep_hld) > 0:
|
||||
lines.append(" [Holdout] (rep-level means, n_reps={})".format(len(rep_hld)))
|
||||
ci_lo, ci_hi = _ci95(rep_hld)
|
||||
lines.append(
|
||||
f" {'Macro AUC':18s}: {rep_hld.mean():.4f} ± {rep_hld.std():.4f}"
|
||||
f" 95% CI [{ci_lo:.4f}, {ci_hi:.4f}]"
|
||||
)
|
||||
|
||||
lines.append("=" * 60)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(
|
||||
description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
ap.add_argument("--run-root", default="analysis_data/pipeline_10x5",
|
||||
help="Root directory containing rep* sub-directories.")
|
||||
ap.add_argument("--eval-modes", nargs="+",
|
||||
choices=["binary", "multiclass"],
|
||||
default=["binary", "multiclass"])
|
||||
ap.add_argument("--out", default=None,
|
||||
help="Output directory for plots and CSVs "
|
||||
"(default: {run-root}/aggregate).")
|
||||
args = ap.parse_args()
|
||||
|
||||
run_root = Path(args.run_root)
|
||||
out_dir = Path(args.out) if args.out else run_root / "aggregate"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print("Loading fold data...")
|
||||
fold_df = load_all_folds(run_root, args.eval_modes)
|
||||
if fold_df.empty:
|
||||
raise SystemExit("No data loaded — check --run-root.")
|
||||
|
||||
fold_df.to_csv(out_dir / "fold_metrics.csv", index=False)
|
||||
print(f" Saved fold_metrics.csv ({len(fold_df)} rows)")
|
||||
|
||||
rep_df = (fold_df.groupby(["rep", "eval_mode"])
|
||||
[["auc_macro", "acc"] +
|
||||
[c for c in fold_df.columns if c.startswith("auc_class")]]
|
||||
.mean()
|
||||
.reset_index())
|
||||
rep_df.to_csv(out_dir / "rep_metrics.csv", index=False)
|
||||
print(f" Saved rep_metrics.csv ({len(rep_df)} rows)")
|
||||
|
||||
summary_text = _print_summary(fold_df, args.eval_modes)
|
||||
print("\n" + summary_text)
|
||||
(out_dir / "overall_summary.txt").write_text(summary_text + "\n")
|
||||
print(f"\n Saved overall_summary.txt")
|
||||
|
||||
print("\nGenerating plots...")
|
||||
for eval_mode in args.eval_modes:
|
||||
if fold_df[fold_df["eval_mode"] == eval_mode].empty:
|
||||
continue
|
||||
_violin(fold_df, eval_mode, out_dir)
|
||||
_mean_roc(fold_df, eval_mode, run_root, out_dir)
|
||||
_rep_stability(fold_df, eval_mode, out_dir)
|
||||
_holdout_stability(fold_df, eval_mode, out_dir)
|
||||
|
||||
print(f"\nAll outputs written to: {out_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Aggregate raw GradCAM heatmaps across all folds for a run.
|
||||
|
||||
For each combination of (eye, class, correct/incorrect) computes:
|
||||
- mean heatmap
|
||||
- std heatmap
|
||||
- count
|
||||
|
||||
Also computes a scalar per patient: fraction of GradCAM attention mass that
|
||||
falls within the expert-segmented optic disc region (from GT contour files),
|
||||
using the manifest.csv to locate the contour for each patient/eye.
|
||||
|
||||
Outputs
|
||||
-------
|
||||
{out_dir}/mean_heatmaps.npz
|
||||
Keys: {eye}_{class_name}_{correct|incorrect}_{mean|std|count}
|
||||
e.g. OD_Glaucoma_correct_mean shape (224, 224)
|
||||
|
||||
{out_dir}/attention_stats.csv
|
||||
per-patient scalars: patient_id, fold, eye, true_name, pred_name,
|
||||
correct, confidence, disc_frac, entropy
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/explainability/aggregate_gradcam.py \
|
||||
--run-dir analysis_data/pipeline_nocrop \
|
||||
--eval-mode binary \
|
||||
--tower-mode single
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
|
||||
def load_disc_mask(contour_path: Path, orig_size: tuple[int, int],
|
||||
cam_h: int, cam_w: int) -> np.ndarray | None:
|
||||
"""
|
||||
Load a PAPILA disc contour TXT file, polygon-fill at original image
|
||||
dimensions, then resize to (cam_h, cam_w). Returns a bool array or
|
||||
None if the contour cannot be loaded.
|
||||
"""
|
||||
try:
|
||||
arr = np.loadtxt(str(contour_path), dtype=np.float32)
|
||||
except Exception:
|
||||
return None
|
||||
if arr.ndim == 1:
|
||||
arr = arr.reshape(-1, 2)
|
||||
if arr.shape[0] < 3 or arr.shape[1] < 2:
|
||||
return None
|
||||
|
||||
# orig_size is (W, H) as PIL convention
|
||||
img = Image.new("L", orig_size, 0)
|
||||
draw = ImageDraw.Draw(img)
|
||||
draw.polygon([tuple(pt) for pt in arr[:, :2]], fill=1)
|
||||
mask = np.array(img.resize((cam_w, cam_h), Image.NEAREST), dtype=bool)
|
||||
return mask
|
||||
|
||||
|
||||
def build_disc_lookup(manifest_path: Path) -> dict[tuple[int, str], tuple[Path, tuple[int, int]]]:
|
||||
"""
|
||||
Returns {(patient_id_int, eye): (disc_contour_path, (img_W, img_H))}.
|
||||
Only PAPILA rows are included.
|
||||
"""
|
||||
mf = pd.read_csv(manifest_path)
|
||||
lookup: dict[tuple[int, str], tuple[Path, tuple[int, int]]] = {}
|
||||
for _, row in mf.iterrows():
|
||||
sid = str(row["sample_id"])
|
||||
if not sid.startswith("papila_RET"):
|
||||
continue
|
||||
# sample_id: papila_RET002OD or papila_RET002OS
|
||||
suffix = sid[len("papila_RET"):] # e.g. "002OD"
|
||||
eye = suffix[-2:] # "OD" or "OS"
|
||||
pid = int(suffix[:-2]) # 2
|
||||
disc_path = Path(str(row["annotation_disc"]))
|
||||
img_path = Path(str(row["image_path"]))
|
||||
if not disc_path.exists():
|
||||
continue
|
||||
# read original image size once
|
||||
try:
|
||||
with Image.open(img_path) as im:
|
||||
orig_size = im.size # (W, H)
|
||||
except Exception:
|
||||
continue
|
||||
lookup[(pid, eye)] = (disc_path, orig_size)
|
||||
return lookup
|
||||
|
||||
|
||||
def attention_entropy(cam: np.ndarray) -> float:
|
||||
flat = cam.flatten().astype(np.float64)
|
||||
flat = flat / (flat.sum() + 1e-12)
|
||||
return float(-np.sum(flat * np.log(flat + 1e-12)))
|
||||
|
||||
|
||||
def load_fold(gradcam_dir: Path):
|
||||
idx_path = gradcam_dir / "gradcam_index.csv"
|
||||
if not idx_path.exists():
|
||||
return None
|
||||
idx = pd.read_csv(idx_path)
|
||||
records = []
|
||||
for _, row in idx.iterrows():
|
||||
pid = row["patient_id"]
|
||||
for eye in ("OD", "OS"):
|
||||
npy = gradcam_dir / f"patient_{pid}_{eye}_cam.npy"
|
||||
if not npy.exists():
|
||||
continue
|
||||
cam = np.load(npy)
|
||||
records.append({
|
||||
"patient_id": pid,
|
||||
"eye": eye,
|
||||
"true_label": int(row["true_label"]),
|
||||
"true_name": row["true_name"],
|
||||
"pred_label": int(row["pred_label"]),
|
||||
"pred_name": row["pred_name"],
|
||||
"confidence": float(row["confidence"]),
|
||||
"correct": bool(row["correct"]),
|
||||
"cam": cam,
|
||||
})
|
||||
return records
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--run-dir", default="analysis_data/pipeline_nocrop")
|
||||
ap.add_argument("--eval-mode", default="binary")
|
||||
ap.add_argument("--tower-mode", default="single")
|
||||
ap.add_argument("--manifest", default="manifest.csv")
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
run_dir = Path(args.run_dir)
|
||||
mode_dir = run_dir / args.eval_mode / args.tower_mode
|
||||
out_dir = mode_dir / "gradcam_aggregate"
|
||||
if args.out:
|
||||
out_dir = Path(args.out)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ---- build disc mask lookup ----
|
||||
manifest_path = Path(args.manifest)
|
||||
disc_lookup = build_disc_lookup(manifest_path)
|
||||
print(f"Disc mask lookup: {len(disc_lookup)} entries from {manifest_path}")
|
||||
|
||||
# ---- collect all records ----
|
||||
all_records = []
|
||||
stat_rows = []
|
||||
fold_dirs = sorted(
|
||||
[d for d in mode_dir.iterdir() if d.is_dir() and d.name.startswith("fold")],
|
||||
key=lambda p: int(p.name.replace("fold", "")),
|
||||
)
|
||||
if not fold_dirs:
|
||||
print(f"No fold dirs found under {mode_dir}")
|
||||
return
|
||||
|
||||
for fd in fold_dirs:
|
||||
gcam_dir = fd / "explainability" / "gradcam"
|
||||
records = load_fold(gcam_dir)
|
||||
if records is None:
|
||||
print(f" [skip] {fd.name}: no gradcam_index.csv")
|
||||
continue
|
||||
print(f" {fd.name}: {len(records)} eye records")
|
||||
for r in records:
|
||||
r["fold"] = fd.name
|
||||
all_records.append(r)
|
||||
|
||||
if not all_records:
|
||||
print("No records found — re-run explain_fold.py first.")
|
||||
return
|
||||
|
||||
print(f"\nTotal eye records: {len(all_records)}")
|
||||
|
||||
h, w = all_records[0]["cam"].shape
|
||||
|
||||
# ---- per-record stats ----
|
||||
n_missing = 0
|
||||
for r in all_records:
|
||||
cam = r["cam"]
|
||||
total = cam.sum() + 1e-12
|
||||
pid = int(r["patient_id"])
|
||||
eye = r["eye"]
|
||||
|
||||
disc_mask = None
|
||||
key = (pid, eye)
|
||||
if key in disc_lookup:
|
||||
disc_path, orig_size = disc_lookup[key]
|
||||
disc_mask = load_disc_mask(disc_path, orig_size, h, w)
|
||||
if disc_mask is None:
|
||||
n_missing += 1
|
||||
disc_frac = float("nan")
|
||||
else:
|
||||
disc_frac = float(cam[disc_mask].sum() / total)
|
||||
|
||||
stat_rows.append({
|
||||
"patient_id": r["patient_id"],
|
||||
"fold": r["fold"],
|
||||
"eye": r["eye"],
|
||||
"true_name": r["true_name"],
|
||||
"pred_name": r["pred_name"],
|
||||
"correct": r["correct"],
|
||||
"confidence": r["confidence"],
|
||||
"disc_frac": disc_frac,
|
||||
"entropy": attention_entropy(cam),
|
||||
})
|
||||
|
||||
if n_missing:
|
||||
print(f" Warning: {n_missing} records had no disc mask (disc_frac=NaN)")
|
||||
|
||||
stats_df = pd.DataFrame(stat_rows)
|
||||
stats_path = out_dir / "attention_stats.csv"
|
||||
stats_df.to_csv(stats_path, index=False)
|
||||
print(f"Saved attention stats → {stats_path}")
|
||||
|
||||
# ---- mean heatmaps ----
|
||||
npz_arrays = {}
|
||||
groups: dict[tuple, list[np.ndarray]] = {}
|
||||
for r in all_records:
|
||||
key = (r["eye"], r["true_name"], "correct" if r["correct"] else "incorrect")
|
||||
groups.setdefault(key, []).append(r["cam"])
|
||||
for r in all_records:
|
||||
key = (r["eye"], r["true_name"], "all")
|
||||
groups.setdefault(key, []).append(r["cam"])
|
||||
|
||||
for (eye, cls, split), cams in groups.items():
|
||||
stack = np.stack(cams, axis=0)
|
||||
key_base = f"{eye}_{cls}_{split}"
|
||||
npz_arrays[f"{key_base}_mean"] = stack.mean(axis=0).astype(np.float32)
|
||||
npz_arrays[f"{key_base}_std"] = stack.std(axis=0).astype(np.float32)
|
||||
npz_arrays[f"{key_base}_count"] = np.array(len(cams))
|
||||
print(f" {key_base}: N={len(cams)}")
|
||||
|
||||
npz_path = out_dir / "mean_heatmaps.npz"
|
||||
np.savez_compressed(npz_path, **npz_arrays)
|
||||
print(f"Saved mean heatmaps → {npz_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -409,6 +409,7 @@ def run_gradcam(
|
||||
overlay_grid_items: list[
|
||||
tuple[Image.Image | None, Image.Image | None, str, bool]
|
||||
] = []
|
||||
index_rows: list[dict] = []
|
||||
|
||||
model.eval()
|
||||
for batch in loader:
|
||||
@@ -492,6 +493,19 @@ def run_gradcam(
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# Save raw CAM arrays
|
||||
np.save(gradcam_dir / f"patient_{pid}_OD_cam.npy", cam_od)
|
||||
np.save(gradcam_dir / f"patient_{pid}_OS_cam.npy", cam_os)
|
||||
index_rows.append({
|
||||
"patient_id": pid,
|
||||
"true_label": label,
|
||||
"true_name": true_name,
|
||||
"pred_label": pred,
|
||||
"pred_name": pred_name,
|
||||
"confidence": conf,
|
||||
"correct": correct,
|
||||
})
|
||||
|
||||
# Accumulate for summary grid
|
||||
od_overlay = overlay_gradcam(orig_od, cam_od, alpha) if orig_od else None
|
||||
os_overlay = overlay_gradcam(orig_os, cam_os, alpha) if orig_os else None
|
||||
@@ -500,6 +514,16 @@ def run_gradcam(
|
||||
|
||||
gcam.remove()
|
||||
|
||||
# ---- save index CSV ----
|
||||
if index_rows:
|
||||
import csv
|
||||
idx_path = gradcam_dir / "gradcam_index.csv"
|
||||
with idx_path.open("w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=list(index_rows[0].keys()))
|
||||
writer.writeheader()
|
||||
writer.writerows(index_rows)
|
||||
print(f" Index CSV → {idx_path}", flush=True)
|
||||
|
||||
# ---- summary grid: N_patients rows × 2 cols (OD overlay | OS overlay) ----
|
||||
n = len(overlay_grid_items)
|
||||
if n == 0:
|
||||
|
||||
@@ -362,94 +362,114 @@ def run_gradcam(
|
||||
gradcam_dir = out_dir / "gradcam"
|
||||
gradcam_dir.mkdir(exist_ok=True)
|
||||
|
||||
target_layer = get_gradcam_layer(model, backbone)
|
||||
gcam = GradCAM(target_layer)
|
||||
|
||||
num_classes = model.bridge.classifier_fused[-1].out_features
|
||||
manifest_path = gradcam_dir / "gradcam_manifest.csv"
|
||||
skip_overlays = manifest_path.exists()
|
||||
overlay_grid_items = []
|
||||
|
||||
model.eval()
|
||||
for batch in loader:
|
||||
img_od = batch["image_1"].to(device)
|
||||
img_os = batch["image_2"].to(device)
|
||||
meta_od = batch["matrix_1"].to(device)
|
||||
meta_os = batch["matrix_2"].to(device)
|
||||
lbl_raw = batch["label_1"][0]
|
||||
label = int(lbl_raw.item() if isinstance(lbl_raw, torch.Tensor) else lbl_raw)
|
||||
pid = batch["id_1"][0]
|
||||
if skip_overlays:
|
||||
print(" Overlays already exist — skipping computation, loading from disk.", flush=True)
|
||||
manifest_df = pd.read_csv(manifest_path)
|
||||
for _, row in manifest_df.iterrows():
|
||||
pid = str(row["pid"])
|
||||
od_path = gradcam_dir / f"gradcam_od_{pid}.png"
|
||||
os_path = gradcam_dir / f"gradcam_os_{pid}.png"
|
||||
od_ov = Image.open(od_path).convert("RGB") if od_path.exists() else None
|
||||
os_ov = Image.open(os_path).convert("RGB") if os_path.exists() else None
|
||||
overlay_grid_items.append((od_ov, os_ov, str(row["short_lbl"]), bool(row["correct"])))
|
||||
else:
|
||||
target_layer = get_gradcam_layer(model, backbone)
|
||||
gcam = GradCAM(target_layer)
|
||||
manifest_rows = []
|
||||
|
||||
cam_od, pred = gcam.compute(img_od, meta_od, model)
|
||||
cam_os, _ = gcam.compute(img_os, meta_os, model)
|
||||
model.eval()
|
||||
for batch in loader:
|
||||
img_od = batch["image_1"].to(device)
|
||||
img_os = batch["image_2"].to(device)
|
||||
meta_od = batch["matrix_1"].to(device)
|
||||
meta_os = batch["matrix_2"].to(device)
|
||||
lbl_raw = batch["label_1"][0]
|
||||
label = int(lbl_raw.item() if isinstance(lbl_raw, torch.Tensor) else lbl_raw)
|
||||
pid = batch["id_1"][0]
|
||||
|
||||
with torch.no_grad():
|
||||
out_od = model(img_od, meta_od)
|
||||
conf = float(torch.softmax(out_od, dim=1)[0, pred].item())
|
||||
cam_od, pred = gcam.compute(img_od, meta_od, model)
|
||||
cam_os, _ = gcam.compute(img_os, meta_os, model)
|
||||
|
||||
row_od = eval_df[
|
||||
(eval_df["Patient ID"] == int(pid)) & (eval_df["eyeID"] == "OD")
|
||||
]
|
||||
row_os = eval_df[
|
||||
(eval_df["Patient ID"] == int(pid)) & (eval_df["eyeID"] == "OS")
|
||||
]
|
||||
orig_od = (
|
||||
Image.open(data.get_image_path(row_od.iloc[0])).convert("RGB")
|
||||
if len(row_od) else None
|
||||
)
|
||||
orig_os = (
|
||||
Image.open(data.get_image_path(row_os.iloc[0])).convert("RGB")
|
||||
if len(row_os) else None
|
||||
)
|
||||
with torch.no_grad():
|
||||
out_od = model(img_od, meta_od)
|
||||
conf = float(torch.softmax(out_od, dim=1)[0, pred].item())
|
||||
|
||||
true_name = label_name(label, eval_mode)
|
||||
pred_name = label_name(pred, eval_mode)
|
||||
correct = label == pred
|
||||
title = (
|
||||
f"Patient {pid} | True: {true_name} | Pred: {pred_name} "
|
||||
f"| conf={conf:.2f} {'✓' if correct else '✗'}"
|
||||
)
|
||||
row_od = eval_df[
|
||||
(eval_df["Patient ID"] == int(pid)) & (eval_df["eyeID"] == "OD")
|
||||
]
|
||||
row_os = eval_df[
|
||||
(eval_df["Patient ID"] == int(pid)) & (eval_df["eyeID"] == "OS")
|
||||
]
|
||||
orig_od = (
|
||||
Image.open(data.get_image_path(row_od.iloc[0])).convert("RGB")
|
||||
if len(row_od) else None
|
||||
)
|
||||
orig_os = (
|
||||
Image.open(data.get_image_path(row_os.iloc[0])).convert("RGB")
|
||||
if len(row_os) else None
|
||||
)
|
||||
|
||||
fig, axes = plt.subplots(2, 2, figsize=(10, 9))
|
||||
fig.suptitle(title, fontsize=11, fontweight="bold", color="green" if correct else "red")
|
||||
true_name = label_name(label, eval_mode)
|
||||
pred_name = label_name(pred, eval_mode)
|
||||
correct = label == pred
|
||||
title = (
|
||||
f"Patient {pid} | True: {true_name} | Pred: {pred_name} "
|
||||
f"| conf={conf:.2f} {'✓' if correct else '✗'}"
|
||||
)
|
||||
|
||||
if orig_od is not None:
|
||||
axes[0, 0].imshow(orig_od)
|
||||
axes[0, 0].set_title("OD — original", fontsize=9)
|
||||
axes[0, 1].imshow(overlay_gradcam(orig_od, cam_od, alpha))
|
||||
axes[0, 1].set_title("OD — GradCAM", fontsize=9)
|
||||
else:
|
||||
axes[0, 0].set_title("OD — (missing)", fontsize=9)
|
||||
axes[0, 0].axis("off")
|
||||
axes[0, 1].axis("off")
|
||||
fig, axes = plt.subplots(2, 2, figsize=(10, 9))
|
||||
fig.suptitle(title, fontsize=11, fontweight="bold", color="green" if correct else "red")
|
||||
|
||||
if orig_os is not None:
|
||||
axes[1, 0].imshow(orig_os)
|
||||
axes[1, 0].set_title("OS — original", fontsize=9)
|
||||
axes[1, 1].imshow(overlay_gradcam(orig_os, cam_os, alpha))
|
||||
axes[1, 1].set_title("OS — GradCAM", fontsize=9)
|
||||
else:
|
||||
axes[1, 0].set_title("OS — (missing)", fontsize=9)
|
||||
axes[1, 0].axis("off")
|
||||
axes[1, 1].axis("off")
|
||||
if orig_od is not None:
|
||||
axes[0, 0].imshow(orig_od)
|
||||
axes[0, 0].set_title("OD — original", fontsize=9)
|
||||
axes[0, 1].imshow(overlay_gradcam(orig_od, cam_od, alpha))
|
||||
axes[0, 1].set_title("OD — GradCAM", fontsize=9)
|
||||
else:
|
||||
axes[0, 0].set_title("OD — (missing)", fontsize=9)
|
||||
axes[0, 0].axis("off")
|
||||
axes[0, 1].axis("off")
|
||||
|
||||
fig.tight_layout()
|
||||
out_path = gradcam_dir / f"patient_{pid}_OD_OS.png"
|
||||
fig.savefig(out_path, dpi=120)
|
||||
plt.close(fig)
|
||||
print(f" Patient {pid}: {true_name} → {pred_name} ({conf:.2f}) → {out_path.name}", flush=True)
|
||||
if orig_os is not None:
|
||||
axes[1, 0].imshow(orig_os)
|
||||
axes[1, 0].set_title("OS — original", fontsize=9)
|
||||
axes[1, 1].imshow(overlay_gradcam(orig_os, cam_os, alpha))
|
||||
axes[1, 1].set_title("OS — GradCAM", fontsize=9)
|
||||
else:
|
||||
axes[1, 0].set_title("OS — (missing)", fontsize=9)
|
||||
axes[1, 0].axis("off")
|
||||
axes[1, 1].axis("off")
|
||||
|
||||
od_overlay = overlay_gradcam(orig_od, cam_od, alpha) if orig_od else None
|
||||
os_overlay = overlay_gradcam(orig_os, cam_os, alpha) if orig_os else None
|
||||
short_lbl = f"P{pid} {true_name[:3]}→{pred_name[:3]} {'✓' if correct else '✗'}"
|
||||
overlay_grid_items.append((od_overlay, os_overlay, short_lbl, correct))
|
||||
fig.tight_layout()
|
||||
out_path = gradcam_dir / f"patient_{pid}_OD_OS.png"
|
||||
fig.savefig(out_path, dpi=120)
|
||||
plt.close(fig)
|
||||
print(f" Patient {pid}: {true_name} → {pred_name} ({conf:.2f}) → {out_path.name}", flush=True)
|
||||
|
||||
gcam.remove()
|
||||
od_overlay = overlay_gradcam(orig_od, cam_od, alpha) if orig_od else None
|
||||
os_overlay = overlay_gradcam(orig_os, cam_os, alpha) if orig_os else None
|
||||
if od_overlay is not None:
|
||||
od_overlay.save(gradcam_dir / f"gradcam_od_{pid}.png")
|
||||
if os_overlay is not None:
|
||||
os_overlay.save(gradcam_dir / f"gradcam_os_{pid}.png")
|
||||
short_lbl = f"P{pid} {true_name[:3]}→{pred_name[:3]} {'✓' if correct else '✗'}"
|
||||
overlay_grid_items.append((od_overlay, os_overlay, short_lbl, correct))
|
||||
manifest_rows.append({"pid": pid, "short_lbl": short_lbl, "correct": correct})
|
||||
|
||||
gcam.remove()
|
||||
pd.DataFrame(manifest_rows).to_csv(manifest_path, index=False)
|
||||
|
||||
# Always regenerate the summary grid
|
||||
n = len(overlay_grid_items)
|
||||
if n == 0:
|
||||
print(" [Phase 2] No patients to visualise.", flush=True)
|
||||
return
|
||||
|
||||
fig, axes = plt.subplots(n, 2, figsize=(8, n * 3.2 + 0.8))
|
||||
fig, axes = plt.subplots(n, 2, figsize=(8, n * 3.2 + 1.5))
|
||||
if n == 1:
|
||||
axes = axes[np.newaxis, :]
|
||||
fig.suptitle("GradCAM Summary Grid — all holdout patients", fontsize=12)
|
||||
@@ -465,7 +485,7 @@ def run_gradcam(
|
||||
axes[i, 1].imshow(os_ov)
|
||||
axes[i, 1].set_title(f"{lbl}\nOS", fontsize=7, color=color)
|
||||
|
||||
fig.tight_layout()
|
||||
fig.tight_layout(rect=[0, 0, 1, 0.97])
|
||||
grid_path = out_dir / "gradcam_summary_grid.png"
|
||||
fig.savefig(grid_path, dpi=120)
|
||||
plt.close(fig)
|
||||
@@ -496,6 +516,7 @@ def _fusion_event_stats(
|
||||
pm: np.ndarray,
|
||||
split_name: str,
|
||||
out_dir: Path,
|
||||
component_labels: tuple[str, str] = ("img", "md"),
|
||||
) -> dict:
|
||||
"""Compute, save, and plot fusion events for one split. Returns summary dict."""
|
||||
N = len(y_true)
|
||||
@@ -503,6 +524,16 @@ def _fusion_event_stats(
|
||||
print(f" [{split_name}] No samples — skipping.", flush=True)
|
||||
return {}
|
||||
|
||||
a, b = component_labels
|
||||
event_labels = [
|
||||
"full correction\n(both wrong→fused right)",
|
||||
f"{a} assist\n({a} wrong, {b} right→right)",
|
||||
f"{b} assist\n({b} wrong, {a} right→right)",
|
||||
"full error\n(both right→fused wrong)",
|
||||
f"{a} drag\n({a} wrong, {b} right→wrong)",
|
||||
f"{b} drag\n({b} wrong, {a} right→wrong)",
|
||||
]
|
||||
|
||||
pred_f = pf.argmax(axis=1)
|
||||
pred_i = pi.argmax(axis=1)
|
||||
pred_m = pm.argmax(axis=1)
|
||||
@@ -529,7 +560,7 @@ def _fusion_event_stats(
|
||||
counts = [int(m.sum()) for m in event_masks]
|
||||
|
||||
print(f"\n [{split_name}] N={N}", flush=True)
|
||||
for label, count in zip(_EVENT_LABELS, counts):
|
||||
for label, count in zip(event_labels, counts):
|
||||
print(f" {label.replace(chr(10), ' '):55s}: {count}", flush=True)
|
||||
n_corr, n_err = counts[0], counts[3]
|
||||
print(f" full correction/error ratio: {n_corr}/{n_err}", flush=True)
|
||||
@@ -582,11 +613,11 @@ def _fusion_event_stats(
|
||||
axes[0].set_xticklabels(["Positive\nevents", "Negative\nevents"])
|
||||
axes[0].set_ylabel("Count")
|
||||
patches = [mpatches.Patch(color=c, label=l.replace("\n", " "))
|
||||
for c, l in zip(_EVENT_COLORS, _EVENT_LABELS)]
|
||||
for c, l in zip(_EVENT_COLORS, event_labels)]
|
||||
axes[0].legend(handles=patches, fontsize=6, loc="upper right")
|
||||
|
||||
box_data = [conf_delta[m] for m in event_masks if m.sum() > 0]
|
||||
box_labels = [l.split("\n")[0] for m, l in zip(event_masks, _EVENT_LABELS) if m.sum() > 0]
|
||||
box_labels = [l.split("\n")[0] for m, l in zip(event_masks, event_labels) if m.sum() > 0]
|
||||
box_cols = [c for m, c in zip(event_masks, _EVENT_COLORS) if m.sum() > 0]
|
||||
if box_data:
|
||||
bp = axes[1].boxplot(box_data, patch_artist=True, widths=0.5)
|
||||
@@ -595,10 +626,10 @@ def _fusion_event_stats(
|
||||
axes[1].set_xticks(range(1, len(box_labels) + 1))
|
||||
axes[1].set_xticklabels(box_labels, rotation=35, ha="right", fontsize=7)
|
||||
axes[1].axhline(0, color="black", linewidth=0.8, linestyle="--")
|
||||
axes[1].set_ylabel("conf_delta\n(fused − avg(img, md))")
|
||||
axes[1].set_ylabel(f"conf_delta\n(fused − avg({a}, {b}))")
|
||||
axes[1].set_title("Confidence delta by event type")
|
||||
|
||||
for mask, color, label in zip(event_masks, _EVENT_COLORS, _EVENT_LABELS):
|
||||
for mask, color, label in zip(event_masks, _EVENT_COLORS, event_labels):
|
||||
if mask.sum() > 0:
|
||||
axes[2].scatter(conf_i[mask], conf_m[mask], c=color,
|
||||
label=label.split("\n")[0], alpha=0.85, s=45, edgecolors="none")
|
||||
@@ -609,8 +640,8 @@ def _fusion_event_stats(
|
||||
axes[2].scatter(conf_i[concordant_bad], conf_m[concordant_bad],
|
||||
c="darkgrey", alpha=0.4, s=20, edgecolors="none", label="concordant wrong")
|
||||
axes[2].plot([0, 1], [0, 1], "k--", linewidth=0.5, alpha=0.4)
|
||||
axes[2].set_xlabel("conf_img")
|
||||
axes[2].set_ylabel("conf_md")
|
||||
axes[2].set_xlabel(f"conf_{a}")
|
||||
axes[2].set_ylabel(f"conf_{b}")
|
||||
axes[2].set_title("Tower confidence space\ncoloured by fusion event")
|
||||
axes[2].legend(fontsize=6, loc="lower right")
|
||||
|
||||
@@ -711,12 +742,13 @@ def run_fusion_event_analysis(fold_dir: Path, out_dir: Path) -> list[dict]:
|
||||
pm_ens = 0.5 * (pm_od + pm_os)
|
||||
summaries.append(_fusion_event_stats(y_val, pf_ens, pi_ens, pm_ens, "val", out_dir))
|
||||
|
||||
# Fused head (if available): learned bilateral combination vs averaged towers
|
||||
# Fused head (if available): learned bilateral combination vs per-eye bridge outputs
|
||||
fused_head_f = fold_dir / "probs_fused_head.npy"
|
||||
if fused_head_f.exists():
|
||||
pf_head = np.load(fused_head_f)
|
||||
summaries.append(_fusion_event_stats(
|
||||
y_val, pf_head, pi_ens, pm_ens, "val_fused_head", out_dir))
|
||||
y_val, pf_head, pf_od, pf_os, "val_fused_head", out_dir,
|
||||
component_labels=("OD", "OS")))
|
||||
else:
|
||||
print(" Val epoch files not found — skipping val.", flush=True)
|
||||
|
||||
@@ -745,6 +777,95 @@ def run_fusion_event_analysis(fold_dir: Path, out_dir: Path) -> list[dict]:
|
||||
return summaries
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cross-fold MD importance summary plot
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _plot_run_md_importance_summary(run_dir: Path, folds: list) -> None:
|
||||
"""Aggregate per-fold MD permutation importance CSVs into a run-level summary plot."""
|
||||
all_dfs = []
|
||||
for fold_idx, fold_dir, _ in folds:
|
||||
csv_path = fold_dir / "explainability" / "md_permutation_importance.csv"
|
||||
if csv_path.exists():
|
||||
df = pd.read_csv(csv_path)
|
||||
df["fold"] = fold_idx
|
||||
all_dfs.append(df)
|
||||
|
||||
if not all_dfs:
|
||||
print(" [MD summary] No per-fold importance CSVs found — skipping.", flush=True)
|
||||
return
|
||||
|
||||
combined = pd.concat(all_dfs, ignore_index=True)
|
||||
_SPECIAL = {"TOTAL_MD_ABLATION", "GAUSSIAN_NOISE_ABLATION"}
|
||||
feature_rows = combined[~combined["feature"].isin(_SPECIAL)]
|
||||
special_rows = combined[combined["feature"].isin(_SPECIAL)]
|
||||
|
||||
agg = (
|
||||
feature_rows.groupby("feature")["importance"]
|
||||
.agg(["mean", "std"])
|
||||
.reset_index()
|
||||
.rename(columns={"mean": "mean_importance", "std": "std_importance"})
|
||||
.sort_values("mean_importance", ascending=False)
|
||||
.reset_index(drop=True)
|
||||
)
|
||||
special_agg = (
|
||||
special_rows.groupby("feature")["importance"]
|
||||
.agg(["mean", "std"])
|
||||
.reset_index()
|
||||
)
|
||||
|
||||
names = agg["feature"].tolist()
|
||||
imps = agg["mean_importance"].tolist()
|
||||
stds = agg["std_importance"].fillna(0).tolist()
|
||||
colors = ["#e05c5c" if v >= 0 else "#5c9ee0" for v in imps]
|
||||
|
||||
fig, ax = plt.subplots(figsize=(9, max(4, (len(names) + 3) * 0.45)))
|
||||
y_pos = np.arange(len(names))
|
||||
ax.barh(y_pos, imps, xerr=stds, color=colors, ecolor="grey", capsize=3, height=0.6)
|
||||
ax.axhline(len(names) - 0.25, color="grey", linewidth=0.6, linestyle="--")
|
||||
|
||||
special_label_map = {
|
||||
"TOTAL_MD_ABLATION": "ALL MD (permute)",
|
||||
"GAUSSIAN_NOISE_ABLATION": "ALL MD (noise)",
|
||||
}
|
||||
special_colors = {
|
||||
"TOTAL_MD_ABLATION": "#c45ce0",
|
||||
"GAUSSIAN_NOISE_ABLATION": "#e08c2a",
|
||||
}
|
||||
extra_ytick_pos = []
|
||||
extra_ytick_labels = []
|
||||
for i, feat in enumerate(["TOTAL_MD_ABLATION", "GAUSSIAN_NOISE_ABLATION"]):
|
||||
row = special_agg[special_agg["feature"] == feat]
|
||||
if row.empty:
|
||||
continue
|
||||
offset = len(names) + 0.5 + i
|
||||
val, err = float(row["mean"].iloc[0]), float(row["std"].iloc[0])
|
||||
ax.barh(offset, val, xerr=err,
|
||||
color=special_colors[feat] if val >= 0 else "#5c9ee0",
|
||||
ecolor="grey", capsize=3, height=0.6)
|
||||
extra_ytick_pos.append(offset)
|
||||
extra_ytick_labels.append(special_label_map[feat])
|
||||
|
||||
ax.set_yticks(list(y_pos) + extra_ytick_pos)
|
||||
ax.set_yticklabels(names + extra_ytick_labels, fontsize=9)
|
||||
ax.invert_yaxis()
|
||||
ax.axvline(0, color="black", linewidth=0.8)
|
||||
ax.set_xlabel("Mean AUC drop (baseline − permuted)", fontsize=10)
|
||||
ax.set_title(
|
||||
f"MD Tower — Permutation Feature Importance ({len(all_dfs)}-fold summary)\n"
|
||||
f"error bars = std across folds",
|
||||
fontsize=11,
|
||||
)
|
||||
fig.tight_layout()
|
||||
out_path = run_dir / "explainability_md_importance_summary.png"
|
||||
fig.savefig(out_path, dpi=150)
|
||||
plt.close(fig)
|
||||
print(f" MD importance summary → {out_path}", flush=True)
|
||||
|
||||
agg.to_csv(run_dir / "explainability_md_importance_summary.csv", index=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cross-fold fusion summary plot
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -761,6 +882,17 @@ def _plot_cross_fold_fusion_summary(df_sum: pd.DataFrame, run_dir: Path) -> None
|
||||
n_folds = len(grp)
|
||||
fold_ids = grp["fold"].values
|
||||
|
||||
# For the fused_head split the two comparators are OD/OS bridges, not img/md towers
|
||||
comp_a, comp_b = ("OD", "OS") if "fused_head" in split_name else ("img", "md")
|
||||
summary_event_labels = [
|
||||
"full correction\n(both wrong→fused right)",
|
||||
f"{comp_a} assist\n({comp_a} wrong, {comp_b} right→right)",
|
||||
f"{comp_b} assist\n({comp_b} wrong, {comp_a} right→right)",
|
||||
"full error\n(both right→fused wrong)",
|
||||
f"{comp_a} drag\n({comp_a} wrong, {comp_b} right→wrong)",
|
||||
f"{comp_b} drag\n({comp_b} wrong, {comp_a} right→wrong)",
|
||||
]
|
||||
|
||||
# Load all per-fold CSVs for this split to get sample-level data
|
||||
sample_dfs = []
|
||||
for fold_idx in fold_ids:
|
||||
@@ -790,7 +922,7 @@ def _plot_cross_fold_fusion_summary(df_sum: pd.DataFrame, run_dir: Path) -> None
|
||||
axes[0].set_xticklabels(["Positive\nevents", "Negative\nevents"])
|
||||
axes[0].set_ylabel("Count (all folds)")
|
||||
patches = [mpatches.Patch(color=c, label=l.replace("\n", " "))
|
||||
for c, l in zip(_EVENT_COLORS, _EVENT_LABELS)]
|
||||
for c, l in zip(_EVENT_COLORS, summary_event_labels)]
|
||||
axes[0].legend(handles=patches, fontsize=6, loc="upper right")
|
||||
|
||||
# Panel 2: per-fold stacked bar (fold variance)
|
||||
@@ -819,7 +951,7 @@ def _plot_cross_fold_fusion_summary(df_sum: pd.DataFrame, run_dir: Path) -> None
|
||||
axes[2].axhline(0, color="grey", linewidth=0.7)
|
||||
axes[2].set_xticks(x)
|
||||
axes[2].set_xticklabels([f"fold {f}" for f in fold_ids], fontsize=8)
|
||||
axes[2].set_ylabel("conf_delta mean\n(fused − avg(img, md))")
|
||||
axes[2].set_ylabel(f"conf_delta mean\n(fused − avg({comp_a}, {comp_b}))")
|
||||
axes[2].set_title("Confidence delta per fold")
|
||||
axes[2].legend(fontsize=8)
|
||||
|
||||
@@ -829,7 +961,7 @@ def _plot_cross_fold_fusion_summary(df_sum: pd.DataFrame, run_dir: Path) -> None
|
||||
box_data = [sample_df.loc[sample_df["event_type"] == k, "conf_delta"].values
|
||||
for k in key_order]
|
||||
box_labels = [l.split("\n")[0]
|
||||
for k, l in zip(_EVENT_KEYS, _EVENT_LABELS) if k in key_order]
|
||||
for k, l in zip(_EVENT_KEYS, summary_event_labels) if k in key_order]
|
||||
box_cols = [c for k, c in zip(_EVENT_KEYS, _EVENT_COLORS) if k in key_order]
|
||||
if box_data:
|
||||
bp = axes[3].boxplot(box_data, patch_artist=True, widths=0.5)
|
||||
@@ -838,7 +970,7 @@ def _plot_cross_fold_fusion_summary(df_sum: pd.DataFrame, run_dir: Path) -> None
|
||||
axes[3].set_xticks(range(1, len(box_labels) + 1))
|
||||
axes[3].set_xticklabels(box_labels, rotation=35, ha="right", fontsize=7)
|
||||
axes[3].axhline(0, color="black", linewidth=0.8, linestyle="--")
|
||||
axes[3].set_ylabel("conf_delta\n(fused − avg(img, md))")
|
||||
axes[3].set_ylabel(f"conf_delta\n(fused − avg({comp_a}, {comp_b}))")
|
||||
axes[3].set_title("Confidence delta by event type\n(all folds)")
|
||||
|
||||
# Panel 5: tower confidence space scatter (all folds combined)
|
||||
@@ -847,7 +979,7 @@ def _plot_cross_fold_fusion_summary(df_sum: pd.DataFrame, run_dir: Path) -> None
|
||||
for key, color in zip(_EVENT_KEYS, _EVENT_COLORS):
|
||||
sub = sample_df[sample_df["event_type"] == key]
|
||||
if len(sub):
|
||||
label = next(l.split("\n")[0] for k, l in zip(_EVENT_KEYS, _EVENT_LABELS)
|
||||
label = next(l.split("\n")[0] for k, l in zip(_EVENT_KEYS, summary_event_labels)
|
||||
if k == key)
|
||||
axes[4].scatter(sub["conf_img"], sub["conf_md"], c=color,
|
||||
label=label, alpha=0.7, s=30, edgecolors="none")
|
||||
@@ -860,8 +992,8 @@ def _plot_cross_fold_fusion_summary(df_sum: pd.DataFrame, run_dir: Path) -> None
|
||||
axes[4].scatter(sub["conf_img"], sub["conf_md"], c=conc_color,
|
||||
alpha=0.3, s=15, edgecolors="none", label=conc_label)
|
||||
axes[4].plot([0, 1], [0, 1], "k--", linewidth=0.5, alpha=0.4)
|
||||
axes[4].set_xlabel("conf_img")
|
||||
axes[4].set_ylabel("conf_md")
|
||||
axes[4].set_xlabel(f"conf_{comp_a}")
|
||||
axes[4].set_ylabel(f"conf_{comp_b}")
|
||||
axes[4].legend(fontsize=6, loc="lower right")
|
||||
axes[4].set_title("Tower confidence space\n(all folds)")
|
||||
|
||||
@@ -1105,6 +1237,12 @@ def main():
|
||||
del model
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
# ---- Cross-fold MD importance summary ----
|
||||
if not args.no_phase1:
|
||||
print(f"\n{'='*60}", flush=True)
|
||||
print("[explain_run] === Cross-fold MD importance summary ===", flush=True)
|
||||
_plot_run_md_importance_summary(run_dir, folds)
|
||||
|
||||
# ---- Cross-fold Phase 3 summary ----
|
||||
if all_phase3_summaries and not args.no_phase3:
|
||||
print(f"\n{'='*60}", flush=True)
|
||||
|
||||
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Granular disc-attention visualisation.
|
||||
|
||||
Layout (3 rows × N_class cols):
|
||||
Row 0 — disc-centred mean GradCAM patch for CORRECT predictions
|
||||
Row 1 — disc-centred mean GradCAM patch for INCORRECT predictions
|
||||
Row 2 — per-patient strip plot of disc_frac (blue=correct, red=incorrect)
|
||||
|
||||
Disc-centred patches: each patient's CAM is translated and scaled so the GT
|
||||
disc centroid sits at the patch centre before averaging. A dashed white circle
|
||||
marks the average GT disc size. This makes cross-patient averaging meaningful
|
||||
regardless of where the disc sits in the original image.
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/explainability/plot_disc_attention_detail.py \
|
||||
--agg-dir analysis_data/pipeline_nocrop/binary/single/gradcam_aggregate \
|
||||
--manifest manifest.csv
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.patches import Circle
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
|
||||
DISC_SPAN = 5 # patch side = DISC_SPAN × disc diameter
|
||||
OUTPUT_SIZE = 96 # pixel size of each thumbnail
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Disc mask helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_disc_mask(contour_path: Path, orig_size: tuple, cam_h: int, cam_w: int):
|
||||
try:
|
||||
arr = np.loadtxt(str(contour_path), dtype=np.float32)
|
||||
except Exception:
|
||||
return None
|
||||
if arr.ndim == 1:
|
||||
arr = arr.reshape(-1, 2)
|
||||
if arr.shape[0] < 3 or arr.shape[1] < 2:
|
||||
return None
|
||||
img = Image.new("L", orig_size, 0)
|
||||
ImageDraw.Draw(img).polygon([tuple(pt) for pt in arr[:, :2]], fill=1)
|
||||
return np.array(img.resize((cam_w, cam_h), Image.NEAREST), dtype=bool)
|
||||
|
||||
|
||||
def build_disc_lookup(manifest_path: Path) -> dict:
|
||||
mf = pd.read_csv(manifest_path)
|
||||
lookup: dict = {}
|
||||
for _, row in mf.iterrows():
|
||||
sid = str(row["sample_id"])
|
||||
if not sid.startswith("papila_RET"):
|
||||
continue
|
||||
suffix = sid[len("papila_RET"):]
|
||||
eye = suffix[-2:]
|
||||
pid = int(suffix[:-2])
|
||||
disc_path = Path(str(row["annotation_disc"]))
|
||||
img_path = Path(str(row["image_path"]))
|
||||
if not disc_path.exists():
|
||||
continue
|
||||
try:
|
||||
with Image.open(img_path) as im:
|
||||
orig_size = im.size
|
||||
except Exception:
|
||||
continue
|
||||
lookup[(pid, eye)] = (disc_path, orig_size)
|
||||
return lookup
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Disc-centred patch extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def disc_centered_patch(
|
||||
cam: np.ndarray,
|
||||
disc_mask: np.ndarray,
|
||||
span: int = DISC_SPAN,
|
||||
out: int = OUTPUT_SIZE,
|
||||
) -> tuple[np.ndarray | None, float | None]:
|
||||
"""
|
||||
Return (patch, disc_r_out):
|
||||
patch — (out, out) float32 in [0, 1]
|
||||
disc_r_out — disc radius in patch-pixel units (for drawing reference circle)
|
||||
"""
|
||||
if disc_mask is None or disc_mask.sum() == 0:
|
||||
return None, None
|
||||
|
||||
ys, xs = np.where(disc_mask)
|
||||
cy, cx = ys.mean(), xs.mean()
|
||||
disc_r = float(np.sqrt(disc_mask.sum() / np.pi))
|
||||
half = max(1, int(round(span * disc_r / 2)))
|
||||
|
||||
h, w = cam.shape
|
||||
y0, y1 = int(round(cy)) - half, int(round(cy)) + half
|
||||
x0, x1 = int(round(cx)) - half, int(round(cx)) + half
|
||||
|
||||
pt = max(0, -y0); pb = max(0, y1 - h)
|
||||
pl = max(0, -x0); pr = max(0, x1 - w)
|
||||
cam_pad = np.pad(cam, ((pt, pb), (pl, pr)), constant_values=0.0)
|
||||
|
||||
patch = cam_pad[y0 + pt : y1 + pt, x0 + pl : x1 + pl]
|
||||
patch_img = Image.fromarray((np.clip(patch, 0, 1) * 255).astype(np.uint8))
|
||||
patch_out = np.array(patch_img.resize((out, out), Image.BILINEAR)) / 255.0
|
||||
|
||||
disc_r_out = out * disc_r / (2 * half)
|
||||
return patch_out.astype(np.float32), disc_r_out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def load_all_cam_records(mode_dir: Path) -> list[dict]:
|
||||
records = []
|
||||
fold_dirs = sorted(
|
||||
[d for d in mode_dir.iterdir() if d.is_dir() and d.name.startswith("fold")],
|
||||
key=lambda p: int(p.name.replace("fold", "")),
|
||||
)
|
||||
for fd in fold_dirs:
|
||||
gcam_dir = fd / "explainability" / "gradcam"
|
||||
idx_path = gcam_dir / "gradcam_index.csv"
|
||||
if not idx_path.exists():
|
||||
continue
|
||||
idx = pd.read_csv(idx_path)
|
||||
for _, row in idx.iterrows():
|
||||
pid = int(row["patient_id"])
|
||||
for eye in ("OD", "OS"):
|
||||
npy = gcam_dir / f"patient_{pid}_{eye}_cam.npy"
|
||||
if not npy.exists():
|
||||
continue
|
||||
records.append({
|
||||
"patient_id": pid,
|
||||
"eye": eye,
|
||||
"true_name": row["true_name"],
|
||||
"correct": bool(row["correct"]),
|
||||
"cam": np.load(npy),
|
||||
})
|
||||
return records
|
||||
|
||||
|
||||
def build_mean_patches(
|
||||
records: list[dict],
|
||||
disc_lookup: dict,
|
||||
classes: list[str],
|
||||
) -> dict[tuple, tuple]:
|
||||
"""
|
||||
Returns {(cls, split): (mean_patch, mean_disc_r_out, count)}
|
||||
split = 'correct' | 'incorrect'
|
||||
"""
|
||||
buckets: dict[tuple, list] = {}
|
||||
radii: dict[tuple, list] = {}
|
||||
|
||||
for r in records:
|
||||
split = "correct" if r["correct"] else "incorrect"
|
||||
key = (r["true_name"], split)
|
||||
pid, eye = r["patient_id"], r["eye"]
|
||||
if (pid, eye) not in disc_lookup:
|
||||
continue
|
||||
disc_path, orig_size = disc_lookup[(pid, eye)]
|
||||
h, w = r["cam"].shape
|
||||
disc_mask = _load_disc_mask(disc_path, orig_size, h, w)
|
||||
patch, disc_r_out = disc_centered_patch(r["cam"], disc_mask)
|
||||
if patch is None:
|
||||
continue
|
||||
buckets.setdefault(key, []).append(patch)
|
||||
radii.setdefault(key, []).append(disc_r_out)
|
||||
|
||||
return {
|
||||
key: (np.stack(ps).mean(0), float(np.mean(radii[key])), len(ps))
|
||||
for key, ps in buckets.items()
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plotting
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--agg-dir", required=True)
|
||||
ap.add_argument("--manifest", default="manifest.csv")
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
agg_dir = Path(args.agg_dir)
|
||||
mode_dir = agg_dir.parent
|
||||
out_path = Path(args.out) if args.out else agg_dir / "disc_attention_detail.png"
|
||||
|
||||
print("Loading disc lookup…")
|
||||
disc_lookup = build_disc_lookup(Path(args.manifest))
|
||||
print(f" {len(disc_lookup)} entries")
|
||||
|
||||
print("Loading CAM records…")
|
||||
records = load_all_cam_records(mode_dir)
|
||||
print(f" {len(records)} eye records")
|
||||
|
||||
stats = pd.read_csv(agg_dir / "attention_stats.csv")
|
||||
classes = sorted({r["true_name"] for r in records})
|
||||
n_cls = len(classes)
|
||||
print(f"Classes: {classes}")
|
||||
|
||||
print("Building disc-centred mean patches…")
|
||||
mean_patches = build_mean_patches(records, disc_lookup, classes)
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Figure
|
||||
# -----------------------------------------------------------------------
|
||||
corr_colors = {"correct": "steelblue", "incorrect": "tomato"}
|
||||
splits = ["correct", "incorrect"]
|
||||
row_labels = ["Correct", "Incorrect", "Disc fraction\n(strip plot)"]
|
||||
|
||||
fig, axes = plt.subplots(3, n_cls, figsize=(4.2 * n_cls, 13))
|
||||
if n_cls == 1:
|
||||
axes = axes[:, np.newaxis]
|
||||
|
||||
# ---- rows 0 & 1: disc-centred heatmaps ----
|
||||
for ri, split in enumerate(splits):
|
||||
for ci, cls in enumerate(classes):
|
||||
ax = axes[ri, ci]
|
||||
key = (cls, split)
|
||||
if key in mean_patches:
|
||||
mean_patch, disc_r_out, count = mean_patches[key]
|
||||
ax.imshow(mean_patch, cmap="jet", vmin=0, vmax=1, origin="upper",
|
||||
extent=[0, OUTPUT_SIZE, OUTPUT_SIZE, 0])
|
||||
cx = cy = OUTPUT_SIZE / 2
|
||||
ax.add_patch(Circle((cx, cy), disc_r_out,
|
||||
fill=False, edgecolor="white",
|
||||
linewidth=2, linestyle="--"))
|
||||
ax.set_title(f"{cls} | {split}\n(N={count})", fontsize=9)
|
||||
else:
|
||||
ax.text(0.5, 0.5, "no data", ha="center", va="center",
|
||||
transform=ax.transAxes, fontsize=9, color="grey")
|
||||
ax.set_title(f"{cls} | {split}", fontsize=9)
|
||||
ax.axis("off")
|
||||
axes[ri, 0].set_ylabel(row_labels[ri], fontsize=10, labelpad=6)
|
||||
|
||||
# ---- row 2: strip plots ----
|
||||
rng = np.random.default_rng(42)
|
||||
for ci, cls in enumerate(classes):
|
||||
ax = axes[2, ci]
|
||||
sub = stats[stats["true_name"] == cls].dropna(subset=["disc_frac"])
|
||||
|
||||
for xi, split in enumerate(splits):
|
||||
correct_val = (split == "correct")
|
||||
pts = sub[sub["correct"] == correct_val]["disc_frac"].values
|
||||
if len(pts) == 0:
|
||||
continue
|
||||
color = corr_colors[split]
|
||||
jitter = rng.uniform(-0.18, 0.18, size=len(pts))
|
||||
ax.scatter(xi + jitter, pts, color=color, alpha=0.7, s=28, edgecolors="none")
|
||||
ax.hlines(pts.mean(), xi - 0.28, xi + 0.28,
|
||||
colors=color, linewidth=2.5, zorder=5)
|
||||
|
||||
ax.set_xticks([0, 1])
|
||||
ax.set_xticklabels(["Correct", "Incorrect"], fontsize=9)
|
||||
ax.set_xlim(-0.55, 1.55)
|
||||
ax.set_ylim(0, 1)
|
||||
ax.set_title(cls, fontsize=10)
|
||||
ax.grid(axis="y", linestyle="--", alpha=0.3)
|
||||
if ci == 0:
|
||||
ax.set_ylabel("Disc fraction\n(GT disc attention)", fontsize=9)
|
||||
|
||||
fig.suptitle(
|
||||
"Disc-centred GradCAM attention | dashed circle = GT disc boundary",
|
||||
fontsize=12,
|
||||
)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,286 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Visualise aggregated GradCAM heatmaps produced by aggregate_gradcam.py.
|
||||
|
||||
Produces two figures:
|
||||
|
||||
Figure 1 — Mean heatmaps grid
|
||||
Rows: classes (e.g. Normal, Glaucoma)
|
||||
Cols: OD_all | OS_all | OD_correct | OD_incorrect | OS_correct | OS_incorrect
|
||||
|
||||
Figure 2 — Attention stats
|
||||
Panel A: disc_frac distribution per class (violin/box), OD and OS side by side
|
||||
Panel B: entropy distribution per class
|
||||
Panel C: disc_frac correct vs incorrect per class (scatter means + error bars)
|
||||
|
||||
Figure 3 — Disc attention vs correct confidence
|
||||
Scatter of disc_frac vs correct_conf (confidence if correct, 1-confidence if wrong)
|
||||
One panel per class, OD and OS overlaid, Pearson r annotated
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/explainability/plot_gradcam_aggregate.py \
|
||||
--agg-dir analysis_data/pipeline_nocrop/binary/single/gradcam_aggregate
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.patches as mpatches
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def load_agg(agg_dir: Path):
|
||||
npz = np.load(agg_dir / "mean_heatmaps.npz")
|
||||
stats = pd.read_csv(agg_dir / "attention_stats.csv")
|
||||
return npz, stats
|
||||
|
||||
|
||||
def _classes_from_npz(npz) -> list[str]:
|
||||
classes = []
|
||||
for key in npz.files:
|
||||
parts = key.split("_")
|
||||
# key format: {EYE}_{ClassName}_{split}_{stat}
|
||||
# ClassName may be multi-word (e.g. "Glaucoma", "Normal", "Suspect")
|
||||
if parts[-1] == "mean" and parts[-2] == "all" and parts[0] == "OD":
|
||||
classes.append(parts[1])
|
||||
return sorted(set(classes))
|
||||
|
||||
|
||||
def plot_mean_heatmaps(npz, classes: list[str], out_path: Path):
|
||||
eyes = ["OD", "OS"]
|
||||
splits = ["all", "correct", "incorrect"]
|
||||
cols = [(e, s) for e in eyes for s in splits] # 6 columns
|
||||
|
||||
n_rows = len(classes)
|
||||
n_cols = len(cols)
|
||||
fig, axes = plt.subplots(n_rows, n_cols, figsize=(n_cols * 2.8, n_rows * 2.8))
|
||||
if n_rows == 1:
|
||||
axes = axes[np.newaxis, :]
|
||||
|
||||
for r, cls in enumerate(classes):
|
||||
for c, (eye, split) in enumerate(cols):
|
||||
ax = axes[r, c]
|
||||
key = f"{eye}_{cls}_{split}_mean"
|
||||
if key not in npz:
|
||||
ax.axis("off")
|
||||
ax.set_title(f"{eye} {split}\n(no data)", fontsize=7)
|
||||
continue
|
||||
cam = npz[key]
|
||||
count = int(npz.get(f"{eye}_{cls}_{split}_count", np.array(0)))
|
||||
ax.imshow(cam, cmap="jet", vmin=0, vmax=1)
|
||||
ax.axis("off")
|
||||
title = f"{cls} | {eye} {split}\n(N={count})"
|
||||
ax.set_title(title, fontsize=7)
|
||||
|
||||
fig.suptitle("Mean GradCAM heatmaps by class / eye / outcome", fontsize=12)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out_path}")
|
||||
|
||||
|
||||
def plot_attention_stats(stats: pd.DataFrame, classes: list[str], out_path: Path):
|
||||
eyes = ["OD", "OS"]
|
||||
cmap = plt.get_cmap("tab10")
|
||||
class_colors = {cls: cmap(i) for i, cls in enumerate(classes)}
|
||||
|
||||
fig, axes = plt.subplots(1, 3, figsize=(16, 5))
|
||||
|
||||
# ---- Panel A: disc_frac per class × eye ----
|
||||
ax = axes[0]
|
||||
positions = []
|
||||
labels = []
|
||||
data_viol = []
|
||||
tick_pos = []
|
||||
pos = 0
|
||||
for cls in classes:
|
||||
for eye in eyes:
|
||||
sub = stats[(stats["true_name"] == cls) & (stats["eye"] == eye)]["disc_frac"].dropna()
|
||||
data_viol.append(sub.values)
|
||||
positions.append(pos)
|
||||
labels.append(f"{cls[:3]}\n{eye}")
|
||||
tick_pos.append(pos)
|
||||
pos += 1
|
||||
pos += 0.5 # gap between classes
|
||||
|
||||
vp = ax.violinplot(data_viol, positions=positions, showmedians=True, widths=0.7)
|
||||
for i, (pc, cls) in enumerate(zip(vp["bodies"], [c for c in classes for _ in eyes])):
|
||||
pc.set_facecolor(class_colors[cls])
|
||||
pc.set_alpha(0.65)
|
||||
ax.set_xticks(tick_pos)
|
||||
ax.set_xticklabels(labels, fontsize=8)
|
||||
ax.set_ylabel("Disc fraction (attention mass within GT disc mask)")
|
||||
ax.set_title("Disc attention by class")
|
||||
ax.axhline(0.5, color="black", linewidth=0.8, linestyle="--", alpha=0.4)
|
||||
ax.grid(axis="y", linestyle="--", alpha=0.3)
|
||||
|
||||
# ---- Panel B: entropy per class × eye (same layout) ----
|
||||
ax = axes[1]
|
||||
data_ent = []
|
||||
for cls in classes:
|
||||
for eye in eyes:
|
||||
sub = stats[(stats["true_name"] == cls) & (stats["eye"] == eye)]["entropy"].dropna()
|
||||
data_ent.append(sub.values)
|
||||
|
||||
vp2 = ax.violinplot(data_ent, positions=positions, showmedians=True, widths=0.7)
|
||||
for pc, cls in zip(vp2["bodies"], [c for c in classes for _ in eyes]):
|
||||
pc.set_facecolor(class_colors[cls])
|
||||
pc.set_alpha(0.65)
|
||||
ax.set_xticks(tick_pos)
|
||||
ax.set_xticklabels(labels, fontsize=8)
|
||||
ax.set_ylabel("Attention entropy (higher = more diffuse)")
|
||||
ax.set_title("Attention entropy by class")
|
||||
ax.grid(axis="y", linestyle="--", alpha=0.3)
|
||||
|
||||
# ---- Panel C: disc_frac correct vs incorrect, mean ± std ----
|
||||
ax = axes[2]
|
||||
x_ticks = []
|
||||
x_labels = []
|
||||
pos = 0
|
||||
for cls in classes:
|
||||
for eye in eyes:
|
||||
for split, marker, ls in [("correct", "o", "-"), ("incorrect", "X", "--")]:
|
||||
sub = stats[
|
||||
(stats["true_name"] == cls) &
|
||||
(stats["eye"] == eye) &
|
||||
(stats["correct"] == (split == "correct"))
|
||||
]["disc_frac"].dropna()
|
||||
if len(sub) == 0:
|
||||
continue
|
||||
ax.errorbar(
|
||||
pos, sub.mean(), yerr=sub.std(),
|
||||
fmt=marker, color=class_colors[cls], linestyle=ls,
|
||||
capsize=4, markersize=7, alpha=0.85,
|
||||
label=f"{cls[:3]} {eye} {split}" if pos < 4 else "_",
|
||||
)
|
||||
pos += 1
|
||||
x_ticks.append(pos - 1.5)
|
||||
x_labels.append(f"{cls[:3]}\n{eye}")
|
||||
pos += 0.5
|
||||
|
||||
ax.axhline(0.5, color="black", linewidth=0.8, linestyle="--", alpha=0.4)
|
||||
ax.set_ylabel("Disc fraction")
|
||||
ax.set_title("Disc fraction: correct vs incorrect\n(circle=correct, X=incorrect)")
|
||||
ax.grid(axis="y", linestyle="--", alpha=0.3)
|
||||
|
||||
# legend: one patch per class
|
||||
patches = [mpatches.Patch(color=class_colors[c], label=c) for c in classes]
|
||||
patches += [
|
||||
plt.Line2D([0], [0], marker="o", color="grey", label="correct", linestyle="none"),
|
||||
plt.Line2D([0], [0], marker="X", color="grey", label="incorrect", linestyle="none"),
|
||||
]
|
||||
ax.legend(handles=patches, fontsize=7, loc="lower right")
|
||||
|
||||
fig.suptitle("GradCAM attention statistics", fontsize=12)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out_path}")
|
||||
|
||||
|
||||
def plot_disc_attention_correlation(stats: pd.DataFrame, classes: list[str], out_path: Path):
|
||||
"""
|
||||
Scatter disc_frac vs correct_conf per class.
|
||||
|
||||
correct_conf = confidence if correct
|
||||
= 1 - confidence if incorrect
|
||||
|
||||
This asks: does focusing attention on the disc region correlate with
|
||||
the model being more confident about the right answer?
|
||||
"""
|
||||
import scipy.stats as scipy_stats
|
||||
|
||||
stats = stats.copy()
|
||||
stats["correct_conf"] = np.where(
|
||||
stats["correct"],
|
||||
stats["confidence"],
|
||||
1.0 - stats["confidence"],
|
||||
)
|
||||
|
||||
corr_colors = {True: "steelblue", False: "tomato"}
|
||||
corr_labels = {True: "Correct", False: "Incorrect"}
|
||||
is_binary = len(classes) == 2
|
||||
|
||||
n_cls = len(classes)
|
||||
fig, axes = plt.subplots(1, n_cls, figsize=(5 * n_cls, 5), sharey=True)
|
||||
if n_cls == 1:
|
||||
axes = [axes]
|
||||
|
||||
for ax, cls in zip(axes, classes):
|
||||
sub = stats[stats["true_name"] == cls]
|
||||
x_all, y_all = [], []
|
||||
|
||||
for correct_val, color in corr_colors.items():
|
||||
csub = sub[sub["correct"] == correct_val]
|
||||
x = csub["disc_frac"].values
|
||||
y = csub["correct_conf"].values
|
||||
ax.scatter(x, y, marker="o", color=color,
|
||||
alpha=0.75, s=30,
|
||||
label=corr_labels[correct_val],
|
||||
edgecolors="none")
|
||||
x_all.extend(x.tolist())
|
||||
y_all.extend(y.tolist())
|
||||
|
||||
# pooled regression line
|
||||
x_arr = np.array(x_all)
|
||||
y_arr = np.array(y_all)
|
||||
if len(x_arr) >= 3:
|
||||
r, p = scipy_stats.pearsonr(x_arr, y_arr)
|
||||
m, b = np.polyfit(x_arr, y_arr, 1)
|
||||
xs = np.linspace(0, 1, 100)
|
||||
ax.plot(xs, m * xs + b, color="black", linewidth=1.5, linestyle="--", alpha=0.7)
|
||||
p_str = f"p={p:.3f}" if p >= 0.001 else "p<0.001"
|
||||
ax.annotate(f"r={r:+.3f}\n{p_str}", xy=(0.05, 0.93), xycoords="axes fraction",
|
||||
fontsize=9, va="top",
|
||||
bbox=dict(boxstyle="round,pad=0.3", facecolor="white", alpha=0.7))
|
||||
|
||||
if is_binary:
|
||||
ax.axhline(0.5, color="red", linewidth=1.0, linestyle=":",
|
||||
alpha=0.7, label="Decision boundary (0.50)")
|
||||
ax.set_xlim(0, 1)
|
||||
ax.set_xlabel("Disc fraction\n(attention mass within GT disc mask)", fontsize=9)
|
||||
ax.set_title(cls, fontsize=11)
|
||||
ax.set_ylim(-0.02, 1.05)
|
||||
ax.grid(linestyle="--", alpha=0.3)
|
||||
ax.legend(fontsize=8, loc="lower right")
|
||||
|
||||
axes[0].set_ylabel("Correct-class confidence\n(conf if correct, 1−conf if wrong)", fontsize=9)
|
||||
fig.suptitle("Disc attention vs correct-class confidence", fontsize=12)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out_path}")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--agg-dir", required=True,
|
||||
help="Directory produced by aggregate_gradcam.py")
|
||||
ap.add_argument("--out-heatmaps", default=None)
|
||||
ap.add_argument("--out-stats", default=None)
|
||||
ap.add_argument("--out-corr", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
agg_dir = Path(args.agg_dir)
|
||||
out_hm = Path(args.out_heatmaps) if args.out_heatmaps else agg_dir / "mean_heatmaps_plot.png"
|
||||
out_st = Path(args.out_stats) if args.out_stats else agg_dir / "attention_stats_plot.png"
|
||||
out_corr = Path(args.out_corr) if args.out_corr else agg_dir / "disc_attention_correlation.png"
|
||||
|
||||
npz, stats = load_agg(agg_dir)
|
||||
classes = _classes_from_npz(npz)
|
||||
print(f"Classes found: {classes}")
|
||||
print(f"Total eye records in stats: {len(stats)}")
|
||||
|
||||
plot_mean_heatmaps(npz, classes, out_hm)
|
||||
plot_attention_stats(stats, classes, out_st)
|
||||
plot_disc_attention_correlation(stats, classes, out_corr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,355 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Re-evaluate holdout accuracy using two threshold strategies:
|
||||
|
||||
1. acc — current behaviour: maximise raw accuracy on (imbalanced) val set
|
||||
2. youden — Youden's J = sensitivity + specificity − 1 on val set
|
||||
|
||||
For each fold the val probs (already saved) supply the threshold, then the
|
||||
model is re-run on the holdout set to get the actual holdout accuracy under
|
||||
each strategy.
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/reeval_holdout_threshold.py \
|
||||
--run-dir analysis_data/pipeline_10x5 \
|
||||
--eval-mode binary
|
||||
|
||||
# or a single nocrop run:
|
||||
python scripts/output_analysis/reeval_holdout_threshold.py \
|
||||
--run-dir analysis_data/pipeline_nocrop \
|
||||
--eval-mode binary \
|
||||
--fold-seed 42
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
from sklearn.metrics import balanced_accuracy_score, roc_auc_score, roc_curve
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from classes.v2.metrics import tune_multiclass_bias
|
||||
from classes.v2.papila_builders import build_papila_data
|
||||
from classes.v2.loader_factory import filter_bilateral_samples, make_loader
|
||||
from classes.v2.models import SingleEyeHT, collect_probs_single_components
|
||||
from classes.v2.profiles import build_papila_profile
|
||||
from classes.v2.split_manager import PatientFirstSplitManager
|
||||
from classes.v2.transforms import build_eval_transform
|
||||
from classes.v2.utils import choose_device
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Threshold helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _acc_threshold(y: np.ndarray, p1: np.ndarray) -> float:
|
||||
grid = np.linspace(0.0, 1.0, 1001)
|
||||
best_t, best_acc = 0.5, -1.0
|
||||
for t in grid:
|
||||
acc = float(((p1 >= t).astype(int) == y).mean())
|
||||
if acc > best_acc or (acc == best_acc and abs(t - 0.5) < abs(best_t - 0.5)):
|
||||
best_acc, best_t = acc, float(t)
|
||||
return best_t
|
||||
|
||||
|
||||
def _youden_threshold(y: np.ndarray, p1: np.ndarray) -> float:
|
||||
if len(np.unique(y)) < 2:
|
||||
return 0.5
|
||||
fpr, tpr, thresholds = roc_curve(y, p1)
|
||||
j = tpr + (1.0 - fpr) - 1.0
|
||||
return float(thresholds[np.argmax(j)])
|
||||
|
||||
|
||||
def _apply_threshold(probs: np.ndarray, threshold: float, num_classes: int) -> np.ndarray:
|
||||
if num_classes == 2:
|
||||
return (probs[:, 1] >= threshold).astype(int)
|
||||
# multiclass: not applicable for a single scalar threshold
|
||||
return probs.argmax(axis=1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-fold evaluation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def eval_fold(
|
||||
fold_dir: Path,
|
||||
fold_idx: int,
|
||||
fold_seed: int,
|
||||
eval_mode: str,
|
||||
args,
|
||||
device: torch.device,
|
||||
) -> dict | None:
|
||||
|
||||
checkpoint = fold_dir / "best_single.pt"
|
||||
if not checkpoint.exists():
|
||||
print(f" [skip] {fold_dir}: no best_single.pt")
|
||||
return None
|
||||
|
||||
val_y_path = fold_dir / "y_true.npy"
|
||||
val_p_path = fold_dir / "probs_fused.npy"
|
||||
if not val_y_path.exists() or not val_p_path.exists():
|
||||
print(f" [skip] {fold_dir}: no val probs")
|
||||
return None
|
||||
|
||||
val_y = np.load(val_y_path)
|
||||
val_p = np.load(val_p_path)
|
||||
num_classes = val_p.shape[1]
|
||||
|
||||
# ---- decision boundaries from val ----
|
||||
if num_classes == 2:
|
||||
t_acc = _acc_threshold(val_y, val_p[:, 1])
|
||||
t_youden = _youden_threshold(val_y, val_p[:, 1])
|
||||
else:
|
||||
# multiclass: compare raw-acc-optimised bias vs balanced-acc-optimised bias
|
||||
# raw-acc bias: temporarily swap objective back to raw accuracy
|
||||
from sklearn.metrics import accuracy_score
|
||||
import copy
|
||||
|
||||
def _tune_bias_raw(y, p):
|
||||
c = p.shape[1]
|
||||
bias = np.zeros(c)
|
||||
grid = np.linspace(-1.0, 1.0, 41)
|
||||
for _ in range(2):
|
||||
for k in range(c):
|
||||
best_v, best_acc = bias[k], -1.0
|
||||
old = bias[k]
|
||||
for v in grid:
|
||||
bias[k] = float(v)
|
||||
logits = np.log(np.clip(p, 1e-8, 1.0)) + bias.reshape(1, -1)
|
||||
acc = float((logits.argmax(1) == y).mean())
|
||||
if acc > best_acc or (acc == best_acc and abs(v) < abs(best_v)):
|
||||
best_acc, best_v = acc, float(v)
|
||||
bias[k] = best_v
|
||||
return bias
|
||||
|
||||
bias_raw = _tune_bias_raw(val_y, val_p)
|
||||
bias_bal = tune_multiclass_bias(val_y, val_p) # balanced acc objective
|
||||
|
||||
# ---- reconstruct holdout split ----
|
||||
data = build_papila_data(
|
||||
image_dir=args.image_dir,
|
||||
clinical_dir=args.clinical_dir,
|
||||
label_col=args.label_col,
|
||||
cat_cols=args.cat_cols,
|
||||
n_splits=args.n_splits,
|
||||
random_seed=fold_seed,
|
||||
iop_corr_method=getattr(args, "iop_corr_method", "ratio"),
|
||||
)
|
||||
df_mode = data.df.copy()
|
||||
if eval_mode == "binary":
|
||||
df_mode = df_mode[df_mode[args.label_col].isin([0, 1])].reset_index(drop=True)
|
||||
|
||||
splitter = PatientFirstSplitManager(
|
||||
patient_col="Patient ID", label_col=args.label_col
|
||||
)
|
||||
split_args = SimpleNamespace(
|
||||
eval_mode=eval_mode,
|
||||
holdout_per_class=args.holdout_per_class,
|
||||
holdout_seed=args.holdout_seed,
|
||||
n_splits=args.n_splits,
|
||||
fold_seed=fold_seed,
|
||||
)
|
||||
plans = splitter.build_plans(
|
||||
clinical=SimpleNamespace(df=df_mode, label_col=args.label_col),
|
||||
args=split_args,
|
||||
profile=None,
|
||||
)
|
||||
split = plans[fold_idx]
|
||||
|
||||
if split.holdout is None or split.holdout.empty:
|
||||
print(f" [skip] {fold_dir}: no holdout")
|
||||
return None
|
||||
|
||||
# ---- build holdout loader ----
|
||||
profile_patient = build_papila_profile(
|
||||
patient_col="Patient ID", label_col=args.label_col, sample_mode="patient"
|
||||
)
|
||||
holdout_samples = filter_bilateral_samples(
|
||||
profile_patient.build_samples(df=split.holdout, clinical=data)
|
||||
)
|
||||
if not holdout_samples:
|
||||
print(f" [skip] {fold_dir}: no bilateral holdout samples")
|
||||
return None
|
||||
|
||||
eval_transform = build_eval_transform(args.backbone)
|
||||
holdout_loader = make_loader(
|
||||
holdout_samples,
|
||||
profile_patient.slot_descriptors(),
|
||||
image_transform=eval_transform,
|
||||
image_preprocessor=None,
|
||||
batch_size=args.batch_size,
|
||||
shuffle=False,
|
||||
num_workers=args.num_workers,
|
||||
)
|
||||
|
||||
# ---- load model ----
|
||||
model = SingleEyeHT(
|
||||
backbone=args.backbone,
|
||||
freeze_ratio=0.0,
|
||||
augment=False,
|
||||
clinical_data=data,
|
||||
num_classes=num_classes,
|
||||
md_hidden_dim=getattr(args, "md_hidden_dim", 64),
|
||||
fusion_dim=getattr(args, "fusion_dim", 128),
|
||||
bridge_mode=getattr(args, "bridge_mode", "fused"),
|
||||
).to(device)
|
||||
model.load_state_dict(
|
||||
torch.load(checkpoint, map_location=device, weights_only=False)
|
||||
)
|
||||
model.eval()
|
||||
|
||||
# ---- run inference ----
|
||||
hld_y, hld_p, _, _ = collect_probs_single_components(
|
||||
model, holdout_loader, device, aggregate_patient=True
|
||||
)
|
||||
|
||||
if len(hld_y) == 0:
|
||||
return None
|
||||
|
||||
hld_auc = float(roc_auc_score(
|
||||
hld_y, hld_p[:, 1] if num_classes == 2 else hld_p,
|
||||
multi_class="ovr" if num_classes > 2 else "raise",
|
||||
))
|
||||
|
||||
if num_classes == 2:
|
||||
acc_old = float(((hld_p[:, 1] >= t_acc).astype(int) == hld_y).mean())
|
||||
acc_new = float(((hld_p[:, 1] >= t_youden).astype(int) == hld_y).mean())
|
||||
bacc_old = balanced_accuracy_score(hld_y, (hld_p[:, 1] >= t_acc).astype(int))
|
||||
bacc_new = balanced_accuracy_score(hld_y, (hld_p[:, 1] >= t_youden).astype(int))
|
||||
row_extra = {"t_old": t_acc, "t_new": t_youden}
|
||||
else:
|
||||
logits_raw = np.log(np.clip(hld_p, 1e-8, 1.0)) + bias_raw.reshape(1, -1)
|
||||
logits_bal = np.log(np.clip(hld_p, 1e-8, 1.0)) + bias_bal.reshape(1, -1)
|
||||
preds_raw = logits_raw.argmax(1)
|
||||
preds_bal = logits_bal.argmax(1)
|
||||
acc_old = float((preds_raw == hld_y).mean())
|
||||
acc_new = float((preds_bal == hld_y).mean())
|
||||
bacc_old = balanced_accuracy_score(hld_y, preds_raw)
|
||||
bacc_new = balanced_accuracy_score(hld_y, preds_bal)
|
||||
row_extra = {"bias_raw": bias_raw.tolist(), "bias_bal": bias_bal.tolist()}
|
||||
|
||||
return {
|
||||
"fold_dir": str(fold_dir),
|
||||
"fold": fold_idx,
|
||||
"fold_seed": fold_seed,
|
||||
"hld_auc": hld_auc,
|
||||
"hld_acc_old": acc_old,
|
||||
"hld_acc_new": acc_new,
|
||||
"hld_bacc_old": bacc_old,
|
||||
"hld_bacc_new": bacc_new,
|
||||
"n_holdout": len(hld_y),
|
||||
**row_extra,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--run-dir", required=True,
|
||||
help="e.g. analysis_data/pipeline_10x5 or analysis_data/pipeline_nocrop")
|
||||
ap.add_argument("--eval-mode", default="binary", choices=["binary", "multiclass"])
|
||||
ap.add_argument("--fold-seed", type=int, default=None,
|
||||
help="Override fold seed (for single-rep runs). "
|
||||
"For 10x5, seeds are inferred from rep dir name.")
|
||||
ap.add_argument("--backbone", default="refugelike")
|
||||
ap.add_argument("--n-splits", type=int, default=5)
|
||||
ap.add_argument("--holdout-per-class", type=int, default=5)
|
||||
ap.add_argument("--holdout-seed", type=int, default=123)
|
||||
ap.add_argument("--label-col", default="Diagnosis")
|
||||
ap.add_argument("--cat-cols", nargs="*", default=["Gender"])
|
||||
ap.add_argument("--image-dir", default="Papila/FundusImages")
|
||||
ap.add_argument("--clinical-dir",default="Papila/ClinicalData")
|
||||
ap.add_argument("--iop-corr-method", default="ratio")
|
||||
ap.add_argument("--batch-size", type=int, default=8)
|
||||
ap.add_argument("--num-workers", type=int, default=4)
|
||||
ap.add_argument("--md-hidden-dim", type=int, default=128)
|
||||
ap.add_argument("--fusion-dim", type=int, default=256)
|
||||
ap.add_argument("--bridge-mode", default="fused")
|
||||
ap.add_argument("--device", default="auto")
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
device = choose_device(args.device)
|
||||
run_dir = Path(args.run_dir)
|
||||
out_path = Path(args.out) if args.out else \
|
||||
run_dir / f"reeval_threshold_{args.eval_mode}.csv"
|
||||
|
||||
# Discover fold dirs — supports both flat (fold0..fold4) and
|
||||
# rep-based (rep00/binary/ensemble/fold0) layouts
|
||||
_BASE_SEED = 100
|
||||
_SEED_STRIDE = 100
|
||||
|
||||
fold_jobs: list[tuple[Path, int, int]] = [] # (fold_dir, fold_idx, fold_seed)
|
||||
|
||||
rep_dirs = sorted(run_dir.glob("rep[0-9]*"))
|
||||
if rep_dirs:
|
||||
for rep_dir in rep_dirs:
|
||||
rep_n = int(rep_dir.name.replace("rep", ""))
|
||||
fold_seed = _BASE_SEED + rep_n * _SEED_STRIDE
|
||||
mode_dir = rep_dir / args.eval_mode / "ensemble"
|
||||
if not mode_dir.exists():
|
||||
continue
|
||||
for fd in sorted(mode_dir.glob("fold[0-9]*"), key=lambda p: int(p.name[4:])):
|
||||
fold_jobs.append((fd, int(fd.name[4:]), fold_seed))
|
||||
else:
|
||||
# flat layout
|
||||
mode_dir = run_dir / args.eval_mode / "ensemble"
|
||||
fold_seed = args.fold_seed if args.fold_seed is not None else 42
|
||||
for fd in sorted(mode_dir.glob("fold[0-9]*"), key=lambda p: int(p.name[4:])):
|
||||
fold_jobs.append((fd, int(fd.name[4:]), fold_seed))
|
||||
|
||||
if not fold_jobs:
|
||||
sys.exit(f"No fold directories found under {run_dir}")
|
||||
|
||||
print(f"Found {len(fold_jobs)} folds to re-evaluate")
|
||||
|
||||
rows = []
|
||||
for i, (fold_dir, fold_idx, fold_seed) in enumerate(fold_jobs):
|
||||
print(f"\n[{i+1}/{len(fold_jobs)}] {fold_dir} fold_seed={fold_seed}")
|
||||
row = eval_fold(fold_dir, fold_idx, fold_seed, args.eval_mode, args, device)
|
||||
if row:
|
||||
rows.append(row)
|
||||
print(f" hld_acc(old)={row['hld_acc_old']:.3f} "
|
||||
f"hld_acc(new)={row['hld_acc_new']:.3f} "
|
||||
f"hld_bacc(old)={row['hld_bacc_old']:.3f} "
|
||||
f"hld_bacc(new)={row['hld_bacc_new']:.3f} "
|
||||
f"hld_auc={row['hld_auc']:.3f}")
|
||||
|
||||
if not rows:
|
||||
print("No results.")
|
||||
return
|
||||
|
||||
df = pd.DataFrame(rows)
|
||||
df.to_csv(out_path, index=False)
|
||||
print(f"\nSaved → {out_path}")
|
||||
is_binary = args.eval_mode == "binary"
|
||||
old_label = "acc-threshold" if is_binary else "raw-acc bias"
|
||||
new_label = "Youden-J" if is_binary else "balanced-acc bias"
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Summary ({args.eval_mode}, n={len(df)} folds)")
|
||||
print(f"{'='*60}")
|
||||
print(f" Holdout AUC: {df.hld_auc.mean():.4f} ± {df.hld_auc.std():.4f}")
|
||||
print(f" Holdout acc ({old_label:<18}): {df.hld_acc_old.mean():.4f} ± {df.hld_acc_old.std():.4f}")
|
||||
print(f" Holdout acc ({new_label:<18}): {df.hld_acc_new.mean():.4f} ± {df.hld_acc_new.std():.4f}")
|
||||
print(f" Holdout bacc ({old_label:<18}): {df.hld_bacc_old.mean():.4f} ± {df.hld_bacc_old.std():.4f}")
|
||||
print(f" Holdout bacc ({new_label:<18}): {df.hld_bacc_new.mean():.4f} ± {df.hld_bacc_new.std():.4f}")
|
||||
print(f" Delta acc (new − old): {df.hld_acc_new.mean() - df.hld_acc_old.mean():+.4f}")
|
||||
print(f" Delta bacc (new − old): {df.hld_bacc_new.mean() - df.hld_bacc_old.mean():+.4f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Assemble the 10×5 aggregate comparison panel.
|
||||
|
||||
Layout (3 rows × 2 cols):
|
||||
col 0 = binary, col 1 = multiclass
|
||||
|
||||
row 0: mean ROC curve
|
||||
row 1: rep stability
|
||||
row 2: holdout stability
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/visualizations/build_10x5_aggregate_panel.py \
|
||||
--run-dir analysis_data/pipeline_10x5 \
|
||||
--out analysis_data/pipeline_10x5/aggregate/aggregate_panel.png
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.image as mpimg
|
||||
|
||||
|
||||
# (row, col, rel_path, label)
|
||||
CELLS = [
|
||||
(0, 0, "aggregate/binary_roc_mean.png", "Binary — Mean ROC"),
|
||||
(0, 1, "aggregate/multiclass_roc_mean.png", "Multiclass — Mean ROC"),
|
||||
(1, 0, "aggregate/binary_rep_stability.png", "Binary — Rep stability"),
|
||||
(1, 1, "aggregate/multiclass_rep_stability.png", "Multiclass — Rep stability"),
|
||||
(2, 0, "aggregate/binary_holdout_stability.png", "Binary — Holdout stability"),
|
||||
(2, 1, "aggregate/multiclass_holdout_stability.png", "Multiclass — Holdout stability"),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--run-dir", default="analysis_data/pipeline_10x5")
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
run_dir = Path(args.run_dir)
|
||||
out = Path(args.out) if args.out else run_dir / "aggregate" / "aggregate_panel.png"
|
||||
|
||||
fig = plt.figure(figsize=(16, 18))
|
||||
gs = fig.add_gridspec(3, 2, hspace=0.06, wspace=0.04)
|
||||
|
||||
for row, col, rel, label in CELLS:
|
||||
ax = fig.add_subplot(gs[row, col])
|
||||
img = mpimg.imread(str(run_dir / rel))
|
||||
ax.imshow(img)
|
||||
ax.axis("off")
|
||||
ax.set_title(label, fontsize=11, pad=5)
|
||||
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Assemble the 10×5 training dynamics panel.
|
||||
|
||||
Layout: 3-row × 4-col gridspec with spanning
|
||||
|
||||
Top 2×2 (each cell spans 2 cols):
|
||||
row 0, cols 0-1: binary early stopping sweep
|
||||
row 0, cols 2-3: multiclass early stopping sweep
|
||||
row 1, cols 0-1: binary cost of stopping early
|
||||
row 1, cols 2-3: multiclass cost of stopping early
|
||||
|
||||
Bottom row of 4 (one col each):
|
||||
row 2, col 0: binary holdout AUC by epoch
|
||||
row 2, col 1: binary val−holdout gap (val-adjusted)
|
||||
row 2, col 2: multiclass holdout AUC by epoch
|
||||
row 2, col 3: multiclass val−holdout gap (val-adjusted)
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/visualizations/build_10x5_training_panel.py \
|
||||
--run-dir analysis_data/pipeline_10x5 \
|
||||
--out analysis_data/pipeline_10x5/aggregate/training_dynamics_panel.png
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.image as mpimg
|
||||
|
||||
|
||||
# (row, col_start, col_end, rel_path, label)
|
||||
# col_end is exclusive slice — use None for single cell
|
||||
CELLS = [
|
||||
# ---- top 2×2: early stopping (each spans 2 cols) ----
|
||||
(0, 0, 2, "binary/ensemble/plots/early_stopping_sweep_fused.png",
|
||||
"Binary — Early stopping sweep"),
|
||||
(0, 2, 4, "multiclass/ensemble/plots/early_stopping_sweep_fused.png",
|
||||
"Multiclass — Early stopping sweep"),
|
||||
(1, 0, 2, "binary/ensemble/plots/early_stopping_sweep_fused_inverted.png",
|
||||
"Binary — Cost of stopping early"),
|
||||
(1, 2, 4, "multiclass/ensemble/plots/early_stopping_sweep_fused_inverted_tol0.002.png",
|
||||
"Multiclass — Cost of stopping early (CI tol=0.002)"),
|
||||
# ---- bottom row of 4: holdout epoch curves (single col each) ----
|
||||
(2, 0, 1, "binary/ensemble/plots/holdout_epoch_curves_fused.png",
|
||||
"Binary — Holdout AUC by epoch"),
|
||||
(2, 1, 2, "binary/ensemble/plots/holdout_epoch_curves_fused_delta_adj.png",
|
||||
"Binary — Val−Holdout gap (val-adjusted)"),
|
||||
(2, 2, 3, "multiclass/ensemble/plots/holdout_epoch_curves_fused.png",
|
||||
"Multiclass — Holdout AUC by epoch"),
|
||||
(2, 3, 4, "multiclass/ensemble/plots/holdout_epoch_curves_fused_delta_adj.png",
|
||||
"Multiclass — Val−Holdout gap (val-adjusted)"),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--run-dir", default="analysis_data/pipeline_10x5")
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
run_dir = Path(args.run_dir)
|
||||
out = Path(args.out) if args.out else run_dir / "aggregate" / "training_dynamics_panel.png"
|
||||
|
||||
fig = plt.figure(figsize=(22, 16))
|
||||
gs = fig.add_gridspec(3, 4, height_ratios=[1, 1, 0.75], hspace=0.08, wspace=0.04)
|
||||
|
||||
for row, col_start, col_end, rel, label in CELLS:
|
||||
ax = fig.add_subplot(gs[row, col_start:col_end])
|
||||
img = mpimg.imread(str(run_dir / rel))
|
||||
ax.imshow(img)
|
||||
ax.axis("off")
|
||||
ax.set_title(label, fontsize=11, pad=5)
|
||||
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Assemble a 2x2 fusion-head comparison panel from pipeline_nocrop.
|
||||
|
||||
Layout:
|
||||
[binary ensemble ROC] [binary fusion-head explainability summary]
|
||||
[multiclass ensemble ROC][multiclass fusion-head explainability summary]
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/visualizations/build_fusion_head_panel.py \
|
||||
--run-dir analysis_data/pipeline_nocrop \
|
||||
--out analysis_data/pipeline_nocrop/fusion_head_comparison_panel.png
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.image as mpimg
|
||||
import numpy as np
|
||||
|
||||
|
||||
ROC_CELLS = [
|
||||
# (row, col, rel_path, label)
|
||||
(0, 0, "binary/ensemble/plots/roc_probs_fused_mean_ovr.png",
|
||||
"Binary — Ensemble"),
|
||||
(0, 1, "binary/ensemble/plots/roc_probs_fused_head_mean_ovr.png",
|
||||
"Binary — Fusion Head"),
|
||||
(1, 0, "multiclass/ensemble/plots/roc_probs_fused_mean_ovr.png",
|
||||
"Multiclass — Ensemble"),
|
||||
(1, 1, "multiclass/ensemble/plots/roc_probs_fused_head_mean_ovr.png",
|
||||
"Multiclass — Fusion Head"),
|
||||
]
|
||||
|
||||
EXPL_ROWS = [
|
||||
# (row_in_grid, rel_path, label)
|
||||
(2, "binary/ensemble/explainability_fusion_summary_val_fused_head.png",
|
||||
"Binary — Fusion Head events (val)"),
|
||||
(3, "multiclass/ensemble/explainability_fusion_summary_val_fused_head.png",
|
||||
"Multiclass — Fusion Head events (val)"),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--run-dir", default="analysis_data/pipeline_nocrop")
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
run_dir = Path(args.run_dir)
|
||||
out = Path(args.out) if args.out else run_dir / "fusion_head_comparison_panel.png"
|
||||
|
||||
# load all images
|
||||
roc_imgs = {(r, c): (mpimg.imread(str(run_dir / rel)), lbl)
|
||||
for r, c, rel, lbl in ROC_CELLS}
|
||||
expl_imgs = [(mpimg.imread(str(run_dir / rel)), lbl)
|
||||
for _, rel, lbl in EXPL_ROWS]
|
||||
|
||||
# 4-row grid: rows 0-1 are the 2×2 ROC square; rows 2-3 are full-width explainability
|
||||
fig = plt.figure(figsize=(14, 20))
|
||||
gs = fig.add_gridspec(
|
||||
4, 2,
|
||||
height_ratios=[1, 1, 0.6, 0.6],
|
||||
hspace=0.06,
|
||||
wspace=0.04,
|
||||
)
|
||||
|
||||
# ROC cells (2×2)
|
||||
for row, col, _, _ in ROC_CELLS:
|
||||
ax = fig.add_subplot(gs[row, col])
|
||||
img, label = roc_imgs[(row, col)]
|
||||
ax.imshow(img)
|
||||
ax.axis("off")
|
||||
ax.set_title(label, fontsize=11, pad=5)
|
||||
|
||||
# Explainability rows (span both columns)
|
||||
for grid_row, (img, label) in zip([2, 3], expl_imgs):
|
||||
ax = fig.add_subplot(gs[grid_row, :])
|
||||
ax.imshow(img)
|
||||
ax.axis("off")
|
||||
ax.set_title(label, fontsize=11, pad=5)
|
||||
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Assemble a metadata-explainability comparison panel.
|
||||
|
||||
Layout (2 rows × 4 cols):
|
||||
row 0 = binary, row 1 = multiclass
|
||||
|
||||
col 0: single md_importance
|
||||
col 1: ensemble md_importance
|
||||
col 2: nocrop ROC
|
||||
col 3: excl_phakic_axial ROC
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/visualizations/build_md_explainability_panel.py \
|
||||
--nocrop-dir analysis_data/pipeline_nocrop \
|
||||
--excl-dir analysis_data/pipeline_nocrop_excl_phakic_axial \
|
||||
--out analysis_data/pipeline_nocrop/md_explainability_panel.png
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.image as mpimg
|
||||
|
||||
|
||||
# (row, col, dir_key, rel_path, label)
|
||||
# dir_key: "nocrop" or "excl"
|
||||
CELLS = [
|
||||
# ---- binary row (row 0) ----
|
||||
(0, 0, "nocrop", "binary/single/explainability_md_importance_summary.png",
|
||||
"Binary — Single"),
|
||||
(0, 1, "nocrop", "binary/ensemble/explainability_md_importance_summary.png",
|
||||
"Binary — Ensemble"),
|
||||
(0, 2, "nocrop", "binary/ensemble/plots/roc_probs_fused_mean_ovr.png",
|
||||
"Binary — nocrop ROC"),
|
||||
(0, 3, "excl", "binary/ensemble/plots/roc_probs_fused_mean_ovr.png",
|
||||
"Binary — excl phakic+axial ROC"),
|
||||
# ---- multiclass row (row 1) ----
|
||||
(1, 0, "nocrop", "multiclass/single/explainability_md_importance_summary.png",
|
||||
"Multiclass — Single"),
|
||||
(1, 1, "nocrop", "multiclass/ensemble/explainability_md_importance_summary.png",
|
||||
"Multiclass — Ensemble"),
|
||||
(1, 2, "nocrop", "multiclass/ensemble/plots/roc_probs_fused_mean_ovr.png",
|
||||
"Multiclass — nocrop ROC"),
|
||||
(1, 3, "excl", "multiclass/ensemble/plots/roc_probs_fused_mean_ovr.png",
|
||||
"Multiclass — excl phakic+axial ROC"),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--nocrop-dir", default="analysis_data/pipeline_nocrop")
|
||||
ap.add_argument("--excl-dir", default="analysis_data/pipeline_nocrop_excl_phakic_axial")
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
nocrop_dir = Path(args.nocrop_dir)
|
||||
excl_dir = Path(args.excl_dir)
|
||||
out = Path(args.out) if args.out else nocrop_dir / "md_explainability_panel.png"
|
||||
|
||||
fig = plt.figure(figsize=(24, 12))
|
||||
gs = fig.add_gridspec(
|
||||
2, 4,
|
||||
hspace=0.08,
|
||||
wspace=0.04,
|
||||
)
|
||||
|
||||
dirs = {"nocrop": nocrop_dir, "excl": excl_dir}
|
||||
|
||||
for row, col, dir_key, rel, label in CELLS:
|
||||
ax = fig.add_subplot(gs[row, col])
|
||||
img = mpimg.imread(str(dirs[dir_key] / rel))
|
||||
ax.imshow(img)
|
||||
ax.axis("off")
|
||||
ax.set_title(label, fontsize=11, pad=5)
|
||||
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Simulate early stopping at each epoch N and show what val/holdout AUC
|
||||
you would have gotten if you stopped there.
|
||||
|
||||
For each fold and each candidate stopping epoch N:
|
||||
- Find the epoch <= N with the highest val AUC (checkpoint selection)
|
||||
- Record the val AUC and holdout AUC at that epoch
|
||||
|
||||
Then plot mean ± std across all folds as a function of N.
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/visualizations/plot_early_stopping_sweep.py \
|
||||
--run-dir analysis_data/pipeline_10x5 \
|
||||
--eval-mode binary \
|
||||
--tower-mode ensemble \
|
||||
--head fused
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
HEAD_COL = {
|
||||
"fused": ("ensemble_val_auc", "ensemble_holdout_auc"),
|
||||
"img": ("ensemble_val_auc_img", "ensemble_holdout_auc_img"),
|
||||
"md": ("ensemble_val_auc_md", "ensemble_holdout_auc_md"),
|
||||
"classic": ("classic_val_auc", "classic_holdout_auc"),
|
||||
}
|
||||
|
||||
|
||||
def load_fold_logs(run_dir: Path, eval_mode: str, tower_mode: str,
|
||||
val_col: str, hld_col: str):
|
||||
logs = []
|
||||
for rep_dir in sorted(run_dir.glob("rep*")):
|
||||
mode_dir = rep_dir / eval_mode / tower_mode
|
||||
if not mode_dir.exists():
|
||||
continue
|
||||
for fd in sorted(
|
||||
[d for d in mode_dir.iterdir() if d.is_dir() and d.name.startswith("fold")],
|
||||
key=lambda p: int(p.name.replace("fold", "")),
|
||||
):
|
||||
log = fd / "epoch_log.csv"
|
||||
if not log.exists():
|
||||
continue
|
||||
df = pd.read_csv(log)
|
||||
if val_col not in df.columns or hld_col not in df.columns:
|
||||
continue
|
||||
df = df[["epoch", val_col, hld_col]].dropna()
|
||||
logs.append(df.reset_index(drop=True))
|
||||
return logs
|
||||
|
||||
|
||||
def sweep(logs: list[pd.DataFrame], val_col: str, hld_col: str):
|
||||
max_epoch = max(df["epoch"].max() for df in logs)
|
||||
epochs = np.arange(1, int(max_epoch) + 1)
|
||||
|
||||
val_mat = np.full((len(logs), len(epochs)), np.nan)
|
||||
hld_mat = np.full((len(logs), len(epochs)), np.nan)
|
||||
|
||||
for i, df in enumerate(logs):
|
||||
for j, n in enumerate(epochs):
|
||||
window = df[df["epoch"] <= n]
|
||||
if window.empty:
|
||||
continue
|
||||
best_idx = window[val_col].idxmax()
|
||||
val_mat[i, j] = window.loc[best_idx, val_col]
|
||||
hld_mat[i, j] = window.loc[best_idx, hld_col]
|
||||
|
||||
return epochs, val_mat, hld_mat
|
||||
|
||||
|
||||
def plot(epochs, val_mat, hld_mat, out_path: Path, title: str, inverted: bool = False, ci_tol: float = 0.0):
|
||||
val_mean = np.nanmean(val_mat, axis=0)
|
||||
val_std = np.nanstd(val_mat, axis=0)
|
||||
hld_mean = np.nanmean(hld_mat, axis=0)
|
||||
hld_std = np.nanstd(hld_mat, axis=0)
|
||||
|
||||
if inverted:
|
||||
# compute cost per fold, then aggregate — avoids max-of-mean bias
|
||||
best_val_per_fold = np.nanmax(val_mat, axis=1, keepdims=True) # (n_folds, 1)
|
||||
best_hld_per_fold = np.nanmax(hld_mat, axis=1, keepdims=True)
|
||||
delta_val = best_val_per_fold - val_mat # (n_folds, n_epochs)
|
||||
delta_hld = best_hld_per_fold - hld_mat
|
||||
y_val = np.nanmean(delta_val, axis=0)
|
||||
y_hld = np.nanmean(delta_hld, axis=0)
|
||||
sy_val = np.nanstd(delta_val, axis=0)
|
||||
sy_hld = np.nanstd(delta_hld, axis=0)
|
||||
else:
|
||||
y_val, y_hld = val_mean, hld_mean
|
||||
sy_val, sy_hld = val_std, hld_std
|
||||
|
||||
fig, ax = plt.subplots(figsize=(11, 5))
|
||||
|
||||
if inverted:
|
||||
# faint per-fold lines
|
||||
for i in range(delta_val.shape[0]):
|
||||
ax.plot(epochs, delta_val[i], color="steelblue", linewidth=0.6, alpha=0.18)
|
||||
ax.plot(epochs, delta_hld[i], color="firebrick", linewidth=0.6, alpha=0.18)
|
||||
|
||||
ax.plot(epochs, y_val, color="steelblue", linewidth=2.0,
|
||||
label="Best val − val@N (val cost of stopping early)" if inverted
|
||||
else "Val AUC (best ckpt up to N)")
|
||||
ax.fill_between(epochs, y_val - sy_val, y_val + sy_val, color="steelblue", alpha=0.15)
|
||||
|
||||
ax.plot(epochs, y_hld, color="firebrick", linewidth=2.0,
|
||||
label="Best hld − hld@N (holdout cost of stopping early)" if inverted
|
||||
else "Holdout AUC (at best val ckpt)")
|
||||
ax.fill_between(epochs, y_hld - sy_hld, y_hld + sy_hld, color="firebrick", alpha=0.15)
|
||||
|
||||
if inverted:
|
||||
ax.axhline(0, color="black", linewidth=1.0, linestyle="--", alpha=0.4)
|
||||
|
||||
# CI-crosses-zero regions (with optional tolerance)
|
||||
val_ci_zero = (y_val - sy_val) <= ci_tol
|
||||
hld_ci_zero = (y_hld - sy_hld) <= ci_tol
|
||||
both_ci_zero = val_ci_zero & hld_ci_zero
|
||||
|
||||
ymax = max(np.nanmax(y_val), np.nanmax(y_hld)) * 1.15
|
||||
ax.fill_between(epochs, 0, ymax, where=val_ci_zero,
|
||||
color="steelblue", alpha=0.12, label="val CI ≤ 0")
|
||||
ax.fill_between(epochs, 0, ymax, where=hld_ci_zero,
|
||||
color="firebrick", alpha=0.12, label="holdout CI ≤ 0")
|
||||
ax.fill_between(epochs, 0, ymax, where=both_ci_zero,
|
||||
color="purple", alpha=0.20, label="both CI ≤ 0")
|
||||
|
||||
ax.set_ylabel("AUC lost vs best achievable")
|
||||
ax.set_ylim(-0.05, ymax)
|
||||
legend_loc = "upper right"
|
||||
else:
|
||||
# gap curve on twin axis
|
||||
gap_mean = val_mean - hld_mean
|
||||
ax2 = ax.twinx()
|
||||
ax2.plot(epochs, gap_mean, color="darkorange", linewidth=1.5,
|
||||
linestyle="--", alpha=0.7, label="Val−Holdout gap")
|
||||
ax2.set_ylabel("Val − Holdout gap", color="darkorange", fontsize=9)
|
||||
ax2.tick_params(axis="y", labelcolor="darkorange")
|
||||
ax2.set_ylim(-0.1, 0.4)
|
||||
lines2, labels2 = ax2.get_legend_handles_labels()
|
||||
|
||||
best_hld_ep = epochs[np.nanargmax(hld_mean)]
|
||||
best_hld_val = hld_mean[np.nanargmax(hld_mean)]
|
||||
ax.axvline(best_hld_ep, color="firebrick", linewidth=1.2, linestyle=":",
|
||||
alpha=0.8, label=f"peak holdout @ epoch {best_hld_ep} ({best_hld_val:.3f})")
|
||||
stable = epochs >= 3
|
||||
min_gap_ep = epochs[stable][np.nanargmin(gap_mean[stable])]
|
||||
ax.axvline(min_gap_ep, color="darkorange", linewidth=1.2, linestyle=":",
|
||||
alpha=0.8, label=f"min gap @ epoch {min_gap_ep}")
|
||||
ax.set_ylabel("AUC")
|
||||
ax.set_ylim(0.5, 1.05)
|
||||
legend_loc = "lower right"
|
||||
|
||||
ax.set_xlabel("Stopping epoch N")
|
||||
ax.set_title(title)
|
||||
ax.grid(axis="y", linestyle="--", alpha=0.35)
|
||||
|
||||
lines1, labels1 = ax.get_legend_handles_labels()
|
||||
if not inverted:
|
||||
lines1 += lines2; labels1 += labels2
|
||||
ax.legend(lines1, labels1, fontsize=8, loc=legend_loc)
|
||||
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=150)
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out_path}")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--run-dir", default="analysis_data/pipeline_10x5")
|
||||
ap.add_argument("--eval-mode", default="binary")
|
||||
ap.add_argument("--tower-mode", default="ensemble")
|
||||
ap.add_argument("--head", default="fused", choices=list(HEAD_COL))
|
||||
ap.add_argument("--out", default=None)
|
||||
ap.add_argument("--inverted", action="store_true",
|
||||
help="Plot best-achievable minus current (cost of stopping early)")
|
||||
ap.add_argument("--ci-tol", type=float, default=0.0,
|
||||
help="Tolerance for CI-crosses-zero shading (default 0.0)")
|
||||
args = ap.parse_args()
|
||||
|
||||
run_dir = Path(args.run_dir)
|
||||
val_col, hld_col = HEAD_COL[args.head]
|
||||
|
||||
logs = load_fold_logs(run_dir, args.eval_mode, args.tower_mode, val_col, hld_col)
|
||||
if not logs:
|
||||
print("No epoch_log.csv files found.")
|
||||
return
|
||||
print(f"Loaded {len(logs)} fold logs")
|
||||
|
||||
epochs, val_mat, hld_mat = sweep(logs, val_col, hld_col)
|
||||
|
||||
tol_tag = f"_tol{args.ci_tol}" if args.ci_tol else ""
|
||||
suffix = f"_inverted{tol_tag}" if args.inverted else ""
|
||||
out = Path(args.out) if args.out else (
|
||||
run_dir / args.eval_mode / args.tower_mode / "plots" /
|
||||
f"early_stopping_sweep_{args.head}{suffix}.png"
|
||||
)
|
||||
title = ("Simulated early stopping — cost of stopping at epoch N\n"
|
||||
if args.inverted else
|
||||
"Simulated early stopping sweep\n")
|
||||
title += f"{run_dir.name} · {args.eval_mode}/{args.tower_mode} · head={args.head}"
|
||||
if args.inverted and args.ci_tol:
|
||||
title += f" (CI tol={args.ci_tol})"
|
||||
plot(epochs, val_mat, hld_mat, out, title, inverted=args.inverted, ci_tol=args.ci_tol)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Plot per-epoch holdout metrics across all folds in a 10x5 (or any multi-rep) run.
|
||||
|
||||
Each fold gets its own line. Lines are coloured by rep.
|
||||
|
||||
Modes
|
||||
-----
|
||||
holdout — raw holdout AUC per epoch (original plot)
|
||||
delta — val_auc - holdout_auc per epoch (generalization gap;
|
||||
closer to 0 = val most faithfully reflects holdout)
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/visualizations/plot_holdout_epoch_curves.py \
|
||||
--run-dir analysis_data/pipeline_10x5 \
|
||||
--eval-mode binary \
|
||||
--tower-mode ensemble \
|
||||
--head fused \
|
||||
--mode delta
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
HEAD_COL = {
|
||||
"fused": ("ensemble_val_auc", "ensemble_holdout_auc"),
|
||||
"img": ("ensemble_val_auc_img", "ensemble_holdout_auc_img"),
|
||||
"md": ("ensemble_val_auc_md", "ensemble_holdout_auc_md"),
|
||||
"classic": ("classic_val_auc", "classic_holdout_auc"),
|
||||
}
|
||||
|
||||
|
||||
def load_curves(run_dir: Path, eval_mode: str, tower_mode: str,
|
||||
val_col: str, hld_col: str, mode: str):
|
||||
"""
|
||||
Returns list of (rep, fold, epochs_array, values_array).
|
||||
mode='holdout' → values = holdout_auc
|
||||
mode='delta' → values = val_auc - holdout_auc
|
||||
"""
|
||||
curves = []
|
||||
for rep_dir in sorted(run_dir.glob("rep*")):
|
||||
mode_dir = rep_dir / eval_mode / tower_mode
|
||||
if not mode_dir.exists():
|
||||
continue
|
||||
fold_dirs = sorted(
|
||||
[d for d in mode_dir.iterdir() if d.is_dir() and d.name.startswith("fold")],
|
||||
key=lambda p: int(p.name.replace("fold", "")),
|
||||
)
|
||||
for fd in fold_dirs:
|
||||
log = fd / "epoch_log.csv"
|
||||
if not log.exists():
|
||||
continue
|
||||
df = pd.read_csv(log)
|
||||
needed = [hld_col] if mode == "holdout" else [val_col, hld_col]
|
||||
if any(c not in df.columns for c in needed):
|
||||
continue
|
||||
df = df.dropna(subset=needed)
|
||||
if mode == "holdout":
|
||||
values = df[hld_col].to_numpy()
|
||||
elif mode == "delta":
|
||||
values = (df[val_col] - df[hld_col]).to_numpy()
|
||||
else: # delta_adj
|
||||
val_arr = df[val_col].to_numpy()
|
||||
hld_arr = df[hld_col].to_numpy()
|
||||
val_best = np.nanmax(val_arr)
|
||||
penalty = val_best - val_arr # 0 when val is at its peak
|
||||
values = (val_arr - hld_arr) + penalty
|
||||
curves.append((rep_dir.name, fd.name, df["epoch"].to_numpy(), values))
|
||||
return curves
|
||||
|
||||
|
||||
def _build_mean_matrix(curves):
|
||||
all_ep = max(len(e) for _, _, e, _ in curves)
|
||||
mat = np.full((len(curves), all_ep), np.nan)
|
||||
for i, (_, _, e, a) in enumerate(curves):
|
||||
mat[i, :len(a)] = a
|
||||
return mat, np.arange(1, all_ep + 1)
|
||||
|
||||
|
||||
def plot(curves, mode: str, out_path: Path, title: str):
|
||||
reps = sorted(set(r for r, _, _, _ in curves))
|
||||
cmap = matplotlib.colormaps.get_cmap("tab10")
|
||||
rep_color = {r: cmap(i / max(len(reps) - 1, 1)) for i, r in enumerate(reps)}
|
||||
|
||||
fig, ax = plt.subplots(figsize=(12, 6))
|
||||
|
||||
for rep, fold, epochs, vals in curves:
|
||||
ax.plot(epochs, vals, color=rep_color[rep], alpha=0.3, linewidth=0.9)
|
||||
|
||||
# per-rep mean
|
||||
for rep in reps:
|
||||
rep_curves = [(e, a) for r, _, e, a in curves if r == rep]
|
||||
max_ep = max(len(e) for e, _ in rep_curves)
|
||||
mat = np.full((len(rep_curves), max_ep), np.nan)
|
||||
for i, (e, a) in enumerate(rep_curves):
|
||||
mat[i, :len(a)] = a
|
||||
mean_curve = np.nanmean(mat, axis=0)
|
||||
ax.plot(np.arange(1, max_ep + 1), mean_curve,
|
||||
color=rep_color[rep], linewidth=1.8, alpha=0.85, label=rep)
|
||||
|
||||
# global mean ± std
|
||||
all_mat, ep_axis = _build_mean_matrix(curves)
|
||||
global_mean = np.nanmean(all_mat, axis=0)
|
||||
global_std = np.nanstd(all_mat, axis=0)
|
||||
ax.plot(ep_axis, global_mean, color="black", linewidth=2.5, zorder=5, label="global mean")
|
||||
ax.fill_between(ep_axis, global_mean - global_std, global_mean + global_std,
|
||||
color="black", alpha=0.12, zorder=4)
|
||||
|
||||
if mode in ("delta", "delta_adj"):
|
||||
ax.axhline(0, color="black", linewidth=1.0, linestyle="--", alpha=0.5)
|
||||
if mode == "delta":
|
||||
ax.set_ylabel("Val AUC − Holdout AUC (gap)")
|
||||
else:
|
||||
ax.set_ylabel("(Val − Holdout) + (ValBest − Val) (adjusted gap)")
|
||||
min_ep = int(ep_axis[np.nanargmin(global_mean)])
|
||||
min_val = global_mean[np.nanargmin(global_mean)]
|
||||
ax.axvline(min_ep, color="red", linewidth=1.2, linestyle=":", alpha=0.7,
|
||||
label=f"min adjusted gap @ epoch {min_ep} ({min_val:+.3f})")
|
||||
else:
|
||||
ax.set_ylabel("Holdout AUC")
|
||||
ax.set_ylim(0, 1.05)
|
||||
|
||||
ax.set_xlabel("Epoch")
|
||||
ax.set_title(title)
|
||||
ax.legend(fontsize=7, ncol=2, loc="upper right" if mode == "delta" else "lower right")
|
||||
ax.grid(axis="y", linestyle="--", alpha=0.4)
|
||||
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=150)
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out_path}")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--run-dir", default="analysis_data/pipeline_10x5")
|
||||
ap.add_argument("--eval-mode", default="binary")
|
||||
ap.add_argument("--tower-mode", default="ensemble")
|
||||
ap.add_argument("--head", default="fused", choices=list(HEAD_COL))
|
||||
ap.add_argument("--mode", default="holdout", choices=["holdout", "delta", "delta_adj"])
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
run_dir = Path(args.run_dir)
|
||||
val_col, hld_col = HEAD_COL[args.head]
|
||||
curves = load_curves(run_dir, args.eval_mode, args.tower_mode,
|
||||
val_col, hld_col, args.mode)
|
||||
|
||||
if not curves:
|
||||
print("No epoch_log.csv files found — check --run-dir / --eval-mode / --tower-mode")
|
||||
return
|
||||
|
||||
print(f"Loaded {len(curves)} fold curves, up to {max(len(e) for _,_,e,_ in curves)} epochs each")
|
||||
|
||||
out = Path(args.out) if args.out else (
|
||||
run_dir / args.eval_mode / args.tower_mode / "plots" /
|
||||
f"holdout_epoch_curves_{args.head}_{args.mode}.png"
|
||||
)
|
||||
label = {"holdout": "holdout AUC", "delta": "val−holdout gap", "delta_adj": "val−holdout gap (val-adjusted)"}[args.mode]
|
||||
title = (f"Per-fold {label} by epoch\n"
|
||||
f"{run_dir.name} · {args.eval_mode}/{args.tower_mode} · head={args.head}")
|
||||
plot(curves, args.mode, out, title)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Per-epoch learning curves for mdonly runs.
|
||||
|
||||
Reads epoch_log.csv from each fold dir and plots val_auc, val_acc,
|
||||
hld_auc, hld_acc — one figure per metric, all folds as individual lines.
|
||||
|
||||
Usage:
|
||||
python scripts/output_analysis/visualizations/plot_mdonly_curves.py \
|
||||
--run-dirs analysis_data/pipeline_mdonly_50ep \
|
||||
analysis_data/pipeline_mdonly_200ep \
|
||||
analysis_data/pipeline_mdonly_500ep \
|
||||
--eval-mode binary
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
METRICS = [
|
||||
("val_auc", "Val AUC"),
|
||||
("val_acc", "Val Accuracy"),
|
||||
("hld_auc", "Holdout AUC"),
|
||||
("hld_acc", "Holdout Accuracy"),
|
||||
]
|
||||
|
||||
PHASE_SHADING = {
|
||||
"tower_warmup": "#d0e8ff",
|
||||
"fused_warmup": "#d0ffe8",
|
||||
}
|
||||
|
||||
|
||||
def _load_folds(mode_dir: Path) -> list[pd.DataFrame]:
|
||||
fold_dirs = sorted(
|
||||
[p for p in mode_dir.glob("fold*") if p.is_dir()],
|
||||
key=lambda p: int(p.name.replace("fold", "")),
|
||||
)
|
||||
frames = []
|
||||
for fd in fold_dirs:
|
||||
csv = fd / "epoch_log.csv"
|
||||
if not csv.exists():
|
||||
print(f" [warn] {csv} not found, skipping")
|
||||
continue
|
||||
df = pd.read_csv(csv)
|
||||
df["_fold"] = int(fd.name.replace("fold", ""))
|
||||
frames.append(df)
|
||||
return frames
|
||||
|
||||
|
||||
def _shade_warmup(ax: plt.Axes, df: pd.DataFrame) -> None:
|
||||
"""Shade warmup phase regions based on first fold's phase column."""
|
||||
if "phase" not in df.columns:
|
||||
return
|
||||
prev_phase = None
|
||||
start = None
|
||||
for _, row in df.iterrows():
|
||||
phase = row["phase"]
|
||||
ep = row["epoch"]
|
||||
if phase != prev_phase:
|
||||
if prev_phase in PHASE_SHADING and start is not None:
|
||||
ax.axvspan(start - 0.5, ep - 0.5, color=PHASE_SHADING[prev_phase],
|
||||
alpha=0.35, zorder=0, label=f"{prev_phase.replace('_', ' ')}")
|
||||
start = ep
|
||||
prev_phase = phase
|
||||
# close last span
|
||||
if prev_phase in PHASE_SHADING and start is not None:
|
||||
ax.axvspan(start - 0.5, df["epoch"].max() + 0.5,
|
||||
color=PHASE_SHADING[prev_phase], alpha=0.35, zorder=0)
|
||||
|
||||
|
||||
def plot_curves(
|
||||
run_dirs: list[Path],
|
||||
eval_mode: str,
|
||||
tower_mode: str,
|
||||
out_dir: Path | None,
|
||||
) -> None:
|
||||
# Collect (label, frames) pairs
|
||||
datasets: list[tuple[str, list[pd.DataFrame]]] = []
|
||||
for rd in run_dirs:
|
||||
mode_dir = rd / eval_mode / tower_mode
|
||||
if not mode_dir.exists():
|
||||
print(f" [skip] {mode_dir} not found")
|
||||
continue
|
||||
frames = _load_folds(mode_dir)
|
||||
if not frames:
|
||||
print(f" [skip] no epoch_log.csv found under {mode_dir}")
|
||||
continue
|
||||
datasets.append((rd.name, frames))
|
||||
|
||||
if not datasets:
|
||||
print("No data found — nothing to plot.")
|
||||
return
|
||||
|
||||
# One figure per metric
|
||||
for metric_key, metric_label in METRICS:
|
||||
# Check any fold actually has this metric with non-nan values
|
||||
has_data = any(
|
||||
not frames[0][metric_key].isna().all()
|
||||
for _, frames in datasets
|
||||
if frames and metric_key in frames[0].columns
|
||||
)
|
||||
if not has_data:
|
||||
continue
|
||||
|
||||
n_runs = len(datasets)
|
||||
fig, axes = plt.subplots(1, n_runs, figsize=(5 * n_runs, 4.5), squeeze=False)
|
||||
|
||||
for col_idx, (run_label, frames) in enumerate(datasets):
|
||||
ax = axes[0][col_idx]
|
||||
if frames and metric_key in frames[0].columns:
|
||||
_shade_warmup(ax, frames[0])
|
||||
|
||||
colours = plt.cm.tab10(np.linspace(0, 0.9, len(frames)))
|
||||
for frame, colour in zip(frames, colours):
|
||||
if metric_key not in frame.columns:
|
||||
continue
|
||||
vals = frame[metric_key].values
|
||||
epochs = frame["epoch"].values
|
||||
mask = ~np.isnan(vals.astype(float))
|
||||
if mask.sum() == 0:
|
||||
continue
|
||||
ax.plot(epochs[mask], vals[mask],
|
||||
linewidth=1.4, color=colour,
|
||||
label=f"fold {frame['_fold'].iloc[0]}")
|
||||
|
||||
ax.set_title(run_label, fontsize=10)
|
||||
ax.set_xlabel("Epoch")
|
||||
if col_idx == 0:
|
||||
ax.set_ylabel(metric_label)
|
||||
ax.legend(fontsize=7, loc="lower right")
|
||||
ax.grid(True, linewidth=0.4, alpha=0.5)
|
||||
|
||||
fig.suptitle(f"{metric_label} [{eval_mode} / {tower_mode}]", fontsize=12)
|
||||
fig.tight_layout()
|
||||
|
||||
dest = out_dir or (run_dirs[0].parent / "mdonly_plots")
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
fname = f"mdonly_{metric_key}_{eval_mode}_{tower_mode}.png"
|
||||
fig.savefig(dest / fname, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {dest / fname}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--run-dirs", nargs="+", required=True,
|
||||
help="One or more run directories (e.g. analysis_data/pipeline_mdonly_50ep).")
|
||||
ap.add_argument("--eval-mode", default="binary", choices=["binary", "multiclass"])
|
||||
ap.add_argument("--tower-mode", default="single", choices=["single", "ensemble"])
|
||||
ap.add_argument("--out", default=None,
|
||||
help="Output directory for plots (default: {first_run_dir}/../mdonly_plots).")
|
||||
args = ap.parse_args()
|
||||
|
||||
run_dirs = [Path(d) for d in args.run_dirs]
|
||||
out_dir = Path(args.out) if args.out else None
|
||||
plot_curves(run_dirs, args.eval_mode, args.tower_mode, out_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -80,22 +80,74 @@ def _fold_colours(n_folds: int) -> dict[int, dict[int, tuple]]:
|
||||
return out
|
||||
|
||||
|
||||
def _load_folds(run_dir: Path, head: str) -> list[pd.DataFrame]:
|
||||
fold_dirs = sorted(
|
||||
def _fold_dirs(run_dir: Path) -> list[Path]:
|
||||
return sorted(
|
||||
[p for p in run_dir.glob("fold*") if p.is_dir() and re.search(r"\d+", p.name)],
|
||||
key=lambda p: int(re.search(r"\d+", p.name).group()),
|
||||
)
|
||||
if not fold_dirs:
|
||||
|
||||
|
||||
def _detect_heads(run_dir: Path) -> list[str]:
|
||||
"""Return all head names available in the first non-empty fold dir."""
|
||||
for fd in _fold_dirs(run_dir):
|
||||
found: list[str] = []
|
||||
# Standard heads come from the predictions CSV — prefer per-eye
|
||||
csv = fd / "predictions_pereye.csv"
|
||||
if not csv.exists():
|
||||
csv = fd / "predictions_classic.csv"
|
||||
if not csv.exists():
|
||||
csv = fd / "predictions.csv"
|
||||
if csv.exists():
|
||||
cols = pd.read_csv(csv, nrows=0).columns.tolist()
|
||||
for h in ["fused", "img", "md"]:
|
||||
if f"prob_{h}_c0" in cols:
|
||||
found.append(h)
|
||||
# Fusion head lives in a separate npy
|
||||
if (fd / "probs_fused_head.npy").exists():
|
||||
found.append("fused_head")
|
||||
if found:
|
||||
return found
|
||||
return ["fused"] # safe fallback
|
||||
|
||||
|
||||
def _load_folds(run_dir: Path, head: str) -> list[pd.DataFrame]:
|
||||
dirs = _fold_dirs(run_dir)
|
||||
if not dirs:
|
||||
raise FileNotFoundError(f"No fold* directories found under {run_dir}")
|
||||
frames = []
|
||||
for fd in fold_dirs:
|
||||
csv = fd / "predictions_classic.csv"
|
||||
for fd in dirs:
|
||||
fold_num = int(re.search(r"\d+", fd.name).group())
|
||||
# fused_head is stored as npy, not in the predictions CSV
|
||||
if head == "fused_head":
|
||||
y_path = fd / "y_true.npy"
|
||||
p_path = fd / "probs_fused_head.npy"
|
||||
if not y_path.exists() or not p_path.exists():
|
||||
print(f" [warn] fused_head npy not found in {fd}, skipping")
|
||||
continue
|
||||
y = np.load(y_path)
|
||||
p = np.load(p_path)
|
||||
df = pd.DataFrame({"y_true": y})
|
||||
for c in range(p.shape[1]):
|
||||
df[f"prob_fused_head_c{c}"] = p[:, c]
|
||||
df["_fold"] = fold_num
|
||||
frames.append(df)
|
||||
continue
|
||||
# Standard heads from predictions CSV — prefer per-eye (2× dots, no OD/OS averaging)
|
||||
csv = fd / "predictions_pereye.csv"
|
||||
if not csv.exists():
|
||||
print(f" [warn] {csv} not found, skipping")
|
||||
csv = fd / "predictions_classic.csv"
|
||||
if not csv.exists():
|
||||
csv = fd / "predictions.csv"
|
||||
if not csv.exists():
|
||||
print(f" [warn] no predictions CSV found in {fd}, skipping")
|
||||
continue
|
||||
df = pd.read_csv(csv)
|
||||
df["_fold"] = int(re.search(r"\d+", fd.name).group())
|
||||
df["_fold"] = fold_num
|
||||
frames.append(df)
|
||||
if not frames:
|
||||
raise FileNotFoundError(
|
||||
f"No data found for head='{head}' in any fold dir under {run_dir}"
|
||||
)
|
||||
return frames
|
||||
|
||||
|
||||
@@ -627,7 +679,9 @@ def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--run-dir", required=True)
|
||||
ap.add_argument("--head", default="fused", choices=["fused", "img", "md"])
|
||||
ap.add_argument("--head", default=None,
|
||||
choices=["fused", "fused_head", "img", "md"],
|
||||
help="Head to plot. Omit to auto-detect and plot all available heads.")
|
||||
ap.add_argument("--style", default="strips",
|
||||
choices=["strips", "confidence", "triangle", "triangle3d"],
|
||||
help="strips: X=true class, Y=P(Glaucoma). "
|
||||
@@ -638,14 +692,22 @@ def main():
|
||||
|
||||
rd = Path(args.run_dir)
|
||||
od = Path(args.out) if args.out else None
|
||||
if args.style == "confidence":
|
||||
plot_confidence(rd, head=args.head, out_dir=od)
|
||||
elif args.style == "triangle":
|
||||
plot_triangle(rd, head=args.head, out_dir=od)
|
||||
elif args.style == "triangle3d":
|
||||
plot_triangle_3d(rd, head=args.head, out_dir=od)
|
||||
else:
|
||||
plot_strip(rd, head=args.head, out_dir=od)
|
||||
|
||||
heads = [args.head] if args.head else _detect_heads(rd)
|
||||
print(f"Heads to plot: {heads}")
|
||||
|
||||
plot_fn = {
|
||||
"confidence": plot_confidence,
|
||||
"triangle": plot_triangle,
|
||||
"triangle3d": plot_triangle_3d,
|
||||
}.get(args.style, plot_strip)
|
||||
|
||||
for head in heads:
|
||||
print(f"\n--- {head} ---")
|
||||
try:
|
||||
plot_fn(rd, head=head, out_dir=od)
|
||||
except Exception as exc:
|
||||
print(f" [skip] {head}: {exc}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -57,12 +57,10 @@ _PROBS_PRIORITY: dict[str, list[str]] = {
|
||||
_ALL_PROBS = ["probs_fused_head", "probs_fused", "probs_bilat", "probs_classic"]
|
||||
|
||||
|
||||
def detect_probs_stem(fold_dir: Path, tower_mode: str | None) -> str | None:
|
||||
def detect_probs_stems(fold_dir: Path, tower_mode: str | None) -> list[str]:
|
||||
"""Return all present probs stems (in priority order) for this fold dir."""
|
||||
priority = _PROBS_PRIORITY.get(tower_mode, _ALL_PROBS) if tower_mode else _ALL_PROBS
|
||||
for stem in priority:
|
||||
if (fold_dir / f"{stem}.npy").exists():
|
||||
return stem
|
||||
return None
|
||||
return [stem for stem in priority if (fold_dir / f"{stem}.npy").exists()]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -246,39 +244,44 @@ def main() -> None:
|
||||
if not fold_dirs:
|
||||
raise SystemExit(f"No fold subdirectories found in {mode_dir}")
|
||||
|
||||
# Determine probs stem
|
||||
probs_stem = args.probs
|
||||
if probs_stem is None:
|
||||
# Determine probs stems to plot
|
||||
if args.probs is not None:
|
||||
stems_to_plot = [args.probs]
|
||||
else:
|
||||
# Collect all stems present across any fold dir
|
||||
seen: list[str] = []
|
||||
for fd in fold_dirs:
|
||||
probs_stem = detect_probs_stem(fd, args.tower_mode)
|
||||
if probs_stem:
|
||||
break
|
||||
if probs_stem is None:
|
||||
raise SystemExit(f"Could not detect a probs file in {mode_dir}/fold*/")
|
||||
print(f"Using probs: {probs_stem}.npy")
|
||||
|
||||
# Load all folds
|
||||
per_fold: list[tuple[int, dict]] = []
|
||||
for fd in fold_dirs:
|
||||
fold_idx = int(fd.name.replace("fold", ""))
|
||||
result = load_fold(fd, probs_stem, args.eval_mode)
|
||||
if result is None:
|
||||
print(f" [skip] fold {fold_idx}: missing y_true or {probs_stem}.npy")
|
||||
continue
|
||||
y, p = result
|
||||
curves = per_class_roc(y, p)
|
||||
per_fold.append((fold_idx, curves))
|
||||
auc_str = " ".join(
|
||||
f"class{k}={v['auc']:.3f}" for k, v in curves.items()
|
||||
)
|
||||
print(f" fold {fold_idx}: {auc_str}")
|
||||
|
||||
if not per_fold:
|
||||
raise SystemExit("No usable folds — nothing to plot.")
|
||||
for s in detect_probs_stems(fd, args.tower_mode):
|
||||
if s not in seen:
|
||||
seen.append(s)
|
||||
stems_to_plot = seen
|
||||
if not stems_to_plot:
|
||||
raise SystemExit(f"Could not detect any probs file in {mode_dir}/fold*/")
|
||||
print(f"Probs stems to plot: {stems_to_plot}")
|
||||
|
||||
out_dir = mode_dir / "plots"
|
||||
plot_perfold(per_fold, out_dir, args.class_names, probs_stem, args.eval_mode)
|
||||
plot_mean_ovr(per_fold, out_dir, args.class_names, probs_stem, args.eval_mode)
|
||||
for probs_stem in stems_to_plot:
|
||||
print(f"\n--- {probs_stem} ---")
|
||||
per_fold: list[tuple[int, dict]] = []
|
||||
for fd in fold_dirs:
|
||||
fold_idx = int(fd.name.replace("fold", ""))
|
||||
result = load_fold(fd, probs_stem, args.eval_mode)
|
||||
if result is None:
|
||||
print(f" [skip] fold {fold_idx}: missing y_true or {probs_stem}.npy")
|
||||
continue
|
||||
y, p = result
|
||||
curves = per_class_roc(y, p)
|
||||
per_fold.append((fold_idx, curves))
|
||||
auc_str = " ".join(f"class{k}={v['auc']:.3f}" for k, v in curves.items())
|
||||
print(f" fold {fold_idx}: {auc_str}")
|
||||
|
||||
if not per_fold:
|
||||
print(f" No usable folds for {probs_stem}, skipping.")
|
||||
continue
|
||||
|
||||
plot_perfold(per_fold, out_dir, args.class_names, probs_stem, args.eval_mode)
|
||||
plot_mean_ovr(per_fold, out_dir, args.class_names, probs_stem, args.eval_mode)
|
||||
|
||||
print(f"\nPlots written to {out_dir}")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user