logging rework temp save
This commit is contained in:
rpotter6298
@@ -197,20 +197,30 @@ def run_permutation_importance(
|
||||
) -> None:
|
||||
print("\n[Phase 1] MD permutation importance ...", flush=True)
|
||||
|
||||
# ---- cache image embeddings + collect meta tensors + labels ----
|
||||
img_feats_list, md_list, label_list = [], [], []
|
||||
# ---- cache bilateral image embeddings + metadata tensors + labels ----
|
||||
img1_feats_list, img2_feats_list = [], []
|
||||
md1_list, md2_list, label_list = [], [], []
|
||||
model.eval()
|
||||
with torch.no_grad():
|
||||
for batch in loader:
|
||||
imgs = batch["image_1"].to(device)
|
||||
meta = batch["matrix_1"].to(device)
|
||||
img1 = batch["image_1"].to(device)
|
||||
img2 = batch["image_2"].to(device)
|
||||
md1 = batch["matrix_1"].to(device)
|
||||
md2 = batch["matrix_2"].to(device)
|
||||
labels = batch["label_1"]
|
||||
img_feats_list.append(model.img_tower(imgs))
|
||||
md_list.append(meta)
|
||||
label_list.append(labels)
|
||||
img1_feats_list.append(model.img_tower(img1))
|
||||
img2_feats_list.append(model.img_tower(img2))
|
||||
md1_list.append(md1)
|
||||
md2_list.append(md2)
|
||||
if isinstance(labels, torch.Tensor):
|
||||
label_list.append(labels)
|
||||
else:
|
||||
label_list.append(torch.tensor(labels, dtype=torch.long))
|
||||
|
||||
img_feats = torch.cat(img_feats_list) # [N, img_dim]
|
||||
md_tensor = torch.cat(md_list) # [N, feature_dim]
|
||||
img1_feats = torch.cat(img1_feats_list) # [N, img_dim]
|
||||
img2_feats = torch.cat(img2_feats_list) # [N, img_dim]
|
||||
md1_tensor = torch.cat(md1_list) # [N, feature_dim]
|
||||
md2_tensor = torch.cat(md2_list) # [N, feature_dim]
|
||||
y_true = torch.cat(label_list).numpy()
|
||||
N = len(y_true)
|
||||
|
||||
@@ -218,11 +228,15 @@ def run_permutation_importance(
|
||||
print(" [Phase 1] No samples — skipping.", flush=True)
|
||||
return
|
||||
|
||||
# ---- baseline AUC ----
|
||||
# ---- baseline AUC (patient-level: average OD/OS fused probabilities) ----
|
||||
with torch.no_grad():
|
||||
md_feats = model.md_tower(md_tensor)
|
||||
fused, _, _ = model.bridge(img_feats, md_feats)
|
||||
probs_baseline = torch.softmax(fused, dim=1).cpu().numpy()
|
||||
md1_feats = model.md_tower(md1_tensor)
|
||||
md2_feats = model.md_tower(md2_tensor)
|
||||
fused1, _, _ = model.bridge(img1_feats, md1_feats)
|
||||
fused2, _, _ = model.bridge(img2_feats, md2_feats)
|
||||
probs_baseline = (
|
||||
0.5 * (torch.softmax(fused1, dim=1) + torch.softmax(fused2, dim=1))
|
||||
).cpu().numpy()
|
||||
_, baseline_auc, _ = _score_arrays(y_true, probs_baseline, num_classes)
|
||||
print(f" Baseline AUC: {baseline_auc:.4f} (N={N})", flush=True)
|
||||
|
||||
@@ -235,13 +249,25 @@ def run_permutation_importance(
|
||||
all_dims = dims["value_dims"] + dims["missing_dims"]
|
||||
drops = []
|
||||
for _ in range(n_permutations):
|
||||
perm = md_tensor.clone()
|
||||
perm1 = md1_tensor.clone()
|
||||
perm2 = md2_tensor.clone()
|
||||
perm_idx = torch.from_numpy(rng.permutation(N)).to(device)
|
||||
perm[:, all_dims] = perm[perm_idx][:, all_dims]
|
||||
# Apply the same donor patient permutation to both eyes to preserve
|
||||
# within-patient coherence while breaking feature-label association.
|
||||
perm1[:, all_dims] = perm1[perm_idx][:, all_dims]
|
||||
perm2[:, all_dims] = perm2[perm_idx][:, all_dims]
|
||||
with torch.no_grad():
|
||||
md_p = model.md_tower(perm)
|
||||
fused_p, _, _ = model.bridge(img_feats, md_p)
|
||||
probs_p = torch.softmax(fused_p, dim=1).cpu().numpy()
|
||||
md1_p = model.md_tower(perm1)
|
||||
md2_p = model.md_tower(perm2)
|
||||
fused1_p, _, _ = model.bridge(img1_feats, md1_p)
|
||||
fused2_p, _, _ = model.bridge(img2_feats, md2_p)
|
||||
probs_p = (
|
||||
0.5
|
||||
* (
|
||||
torch.softmax(fused1_p, dim=1)
|
||||
+ torch.softmax(fused2_p, dim=1)
|
||||
)
|
||||
).cpu().numpy()
|
||||
_, auc_p, _ = _score_arrays(y_true, probs_p, num_classes)
|
||||
drops.append(baseline_auc - auc_p)
|
||||
|
||||
@@ -254,6 +280,56 @@ def run_permutation_importance(
|
||||
|
||||
results.sort(key=lambda r: r["importance"], reverse=True)
|
||||
|
||||
# ---- total MD ablation (all features permuted simultaneously) ----
|
||||
print(" Running total MD ablation ...", flush=True)
|
||||
total_drops = []
|
||||
for _ in range(n_permutations):
|
||||
perm_idx = torch.from_numpy(rng.permutation(N)).to(device)
|
||||
perm1_all = md1_tensor[perm_idx]
|
||||
perm2_all = md2_tensor[perm_idx]
|
||||
with torch.no_grad():
|
||||
md1_all = model.md_tower(perm1_all)
|
||||
md2_all = model.md_tower(perm2_all)
|
||||
f1, _, _ = model.bridge(img1_feats, md1_all)
|
||||
f2, _, _ = model.bridge(img2_feats, md2_all)
|
||||
probs_all = (
|
||||
0.5 * (torch.softmax(f1, dim=1) + torch.softmax(f2, dim=1))
|
||||
).cpu().numpy()
|
||||
_, auc_all, _ = _score_arrays(y_true, probs_all, num_classes)
|
||||
total_drops.append(baseline_auc - auc_all)
|
||||
total_mean = float(np.mean(total_drops))
|
||||
total_std = float(np.std(total_drops))
|
||||
print(
|
||||
f" Total MD ablation Δ AUC = {total_mean:+.4f} ± {total_std:.4f}", flush=True
|
||||
)
|
||||
|
||||
# ---- Gaussian noise ablation (tests architectural vs informational benefit) ----
|
||||
print(" Running Gaussian noise ablation ...", flush=True)
|
||||
noise_drops = []
|
||||
for _ in range(n_permutations):
|
||||
noise1 = torch.randn_like(md1_tensor)
|
||||
noise2 = torch.randn_like(md2_tensor)
|
||||
with torch.no_grad():
|
||||
md1_noise = model.md_tower(noise1)
|
||||
md2_noise = model.md_tower(noise2)
|
||||
f1, _, _ = model.bridge(img1_feats, md1_noise)
|
||||
f2, _, _ = model.bridge(img2_feats, md2_noise)
|
||||
probs_noise = (
|
||||
0.5 * (torch.softmax(f1, dim=1) + torch.softmax(f2, dim=1))
|
||||
).cpu().numpy()
|
||||
_, auc_noise, _ = _score_arrays(y_true, probs_noise, num_classes)
|
||||
noise_drops.append(baseline_auc - auc_noise)
|
||||
noise_mean = float(np.mean(noise_drops))
|
||||
noise_std = float(np.std(noise_drops))
|
||||
print(
|
||||
f" Gaussian noise ablation Δ AUC = {noise_mean:+.4f} ± {noise_std:.4f}", flush=True
|
||||
)
|
||||
print(
|
||||
f" [interpretation] permutation Δ={total_mean:+.4f} noise Δ={noise_mean:+.4f} "
|
||||
f"informational gain = {total_mean - noise_mean:+.4f}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# ---- save CSV ----
|
||||
import csv
|
||||
|
||||
@@ -262,6 +338,8 @@ def run_permutation_importance(
|
||||
writer = csv.DictWriter(f, fieldnames=["feature", "importance", "std"])
|
||||
writer.writeheader()
|
||||
writer.writerows(results)
|
||||
writer.writerow({"feature": "TOTAL_MD_ABLATION", "importance": total_mean, "std": total_std})
|
||||
writer.writerow({"feature": "GAUSSIAN_NOISE_ABLATION", "importance": noise_mean, "std": noise_std})
|
||||
|
||||
# ---- bar chart ----
|
||||
names = [r["feature"] for r in results]
|
||||
@@ -269,13 +347,26 @@ def run_permutation_importance(
|
||||
stds = [r["std"] for r in results]
|
||||
colors = ["#e05c5c" if v >= 0 else "#5c9ee0" for v in imps]
|
||||
|
||||
fig, ax = plt.subplots(figsize=(9, max(4, len(names) * 0.45)))
|
||||
fig, ax = plt.subplots(figsize=(9, max(4, (len(names) + 3) * 0.45)))
|
||||
y_pos = np.arange(len(names))
|
||||
bars = ax.barh(
|
||||
ax.barh(
|
||||
y_pos, imps, xerr=stds, color=colors, ecolor="grey", capsize=3, height=0.6
|
||||
)
|
||||
ax.set_yticks(y_pos)
|
||||
ax.set_yticklabels(names, fontsize=9)
|
||||
ax.axhline(len(names) - 0.25, color="grey", linewidth=0.6, linestyle="--")
|
||||
# total ablation
|
||||
ax.barh(
|
||||
len(names) + 0.5, total_mean, xerr=total_std,
|
||||
color="#c45ce0" if total_mean >= 0 else "#5c9ee0",
|
||||
ecolor="grey", capsize=3, height=0.6,
|
||||
)
|
||||
# gaussian noise ablation
|
||||
ax.barh(
|
||||
len(names) + 1.5, noise_mean, xerr=noise_std,
|
||||
color="#e08c2a" if noise_mean >= 0 else "#5c9ee0",
|
||||
ecolor="grey", capsize=3, height=0.6,
|
||||
)
|
||||
ax.set_yticks(list(y_pos) + [len(names) + 0.5, len(names) + 1.5])
|
||||
ax.set_yticklabels(names + ["ALL MD (permute)", "ALL MD (noise)"], fontsize=9)
|
||||
ax.invert_yaxis()
|
||||
ax.axvline(0, color="black", linewidth=0.8)
|
||||
ax.set_xlabel("Mean AUC drop (baseline − permuted)", fontsize=10)
|
||||
@@ -325,7 +416,8 @@ def run_gradcam(
|
||||
img_os = batch["image_2"].to(device) # [1, 3, H, W]
|
||||
meta_od = batch["matrix_1"].to(device) # [1, feature_dim]
|
||||
meta_os = batch["matrix_2"].to(device)
|
||||
label = int(batch["label_1"][0].item())
|
||||
lbl_raw = batch["label_1"][0]
|
||||
label = int(lbl_raw.item() if isinstance(lbl_raw, torch.Tensor) else lbl_raw)
|
||||
pid = batch["id_1"][0]
|
||||
|
||||
# GradCAM for each eye (OD drives the prediction label)
|
||||
@@ -488,9 +580,149 @@ def parse_args():
|
||||
ap.add_argument("--batch-size", type=int, default=1)
|
||||
ap.add_argument("--no-phase1", action="store_true", help="Skip MD importance")
|
||||
ap.add_argument("--no-phase2", action="store_true", help="Skip GradCAM")
|
||||
ap.add_argument("--no-phase3", action="store_true", help="Skip fusion event analysis")
|
||||
return ap.parse_args()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 3 — Fusion event analysis
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def run_fusion_event_analysis(
|
||||
model: SingleEyeHT,
|
||||
loader,
|
||||
device: torch.device,
|
||||
out_dir: Path,
|
||||
) -> None:
|
||||
print("\n[Phase 3] Fusion event analysis ...", flush=True)
|
||||
|
||||
from classes.v2.models import collect_probs_single_components
|
||||
|
||||
y_true, pf, pi, pm = collect_probs_single_components(
|
||||
model, loader, device, aggregate_patient=True
|
||||
)
|
||||
N = len(y_true)
|
||||
if N == 0:
|
||||
print(" [Phase 3] No samples — skipping.", flush=True)
|
||||
return
|
||||
|
||||
pred_f = pf.argmax(axis=1)
|
||||
pred_i = pi.argmax(axis=1)
|
||||
pred_m = pm.argmax(axis=1)
|
||||
|
||||
corrections = (pred_f == y_true) & (pred_i != y_true) & (pred_m != y_true)
|
||||
errors = (pred_f != y_true) & (pred_i == y_true) & (pred_m == y_true)
|
||||
n_corr = corrections.sum()
|
||||
n_err = errors.sum()
|
||||
both_wrong = ((pred_i != y_true) & (pred_m != y_true)).sum()
|
||||
both_correct = ((pred_i == y_true) & (pred_m == y_true)).sum()
|
||||
|
||||
print(f" N={N} corrections={n_corr} errors={n_err} ratio={n_corr}/{n_err}", flush=True)
|
||||
print(f" correction rate: {n_corr}/{both_wrong} = {n_corr/max(both_wrong,1):.2%} of both-wrong cases", flush=True)
|
||||
print(f" error rate: {n_err}/{both_correct} = {n_err/max(both_correct,1):.2%} of both-correct cases", flush=True)
|
||||
|
||||
# ---- cache intermediate hm/hi vectors for all patients ----
|
||||
model.eval()
|
||||
hm1_list, hm2_list, hi1_list, hi2_list = [], [], [], []
|
||||
with torch.no_grad():
|
||||
for batch in loader:
|
||||
x1 = batch.get("image_1"); m1 = batch.get("matrix_1")
|
||||
x2 = batch.get("image_2"); m2 = batch.get("matrix_2")
|
||||
if not (torch.is_tensor(x1) and torch.is_tensor(m1)):
|
||||
continue
|
||||
hi1 = model.bridge.ln_img(model.bridge.W_img(model.img_tower(x1.to(device))))
|
||||
hi2 = model.bridge.ln_img(model.bridge.W_img(model.img_tower(x2.to(device))))
|
||||
hm1 = model.bridge.ln_md(model.bridge.W_md(model.md_tower(m1.to(device))))
|
||||
hm2 = model.bridge.ln_md(model.bridge.W_md(model.md_tower(m2.to(device))))
|
||||
hi1_list.append(hi1.cpu()); hi2_list.append(hi2.cpu())
|
||||
hm1_list.append(hm1.cpu()); hm2_list.append(hm2.cpu())
|
||||
|
||||
hi1 = torch.cat(hi1_list) # [N, fusion_dim]
|
||||
hi2 = torch.cat(hi2_list)
|
||||
hm1 = torch.cat(hm1_list) # [N, fusion_dim]
|
||||
hm2 = torch.cat(hm2_list)
|
||||
|
||||
hm1_mean = hm1.mean(dim=0, keepdim=True)
|
||||
hm2_mean = hm2.mean(dim=0, keepdim=True)
|
||||
|
||||
# ---- for each patient: compare logit[true_class] with real hm vs mean hm ----
|
||||
gains = []
|
||||
with torch.no_grad():
|
||||
for idx in range(N):
|
||||
true_cls = int(y_true[idx])
|
||||
# patient-level average of OD/OS fused vectors (SE skipped: hard to replicate outside forward)
|
||||
fused_real = (hi1[idx:idx+1] * hm1[idx:idx+1] + hi2[idx:idx+1] * hm2[idx:idx+1]) * 0.5
|
||||
fused_mean = (hi1[idx:idx+1] * hm1_mean + hi2[idx:idx+1] * hm2_mean) * 0.5
|
||||
logit_real = model.bridge.classifier_fused(fused_real.to(device))
|
||||
logit_mean = model.bridge.classifier_fused(fused_mean.to(device))
|
||||
gain = (logit_real[0, true_cls] - logit_mean[0, true_cls]).item()
|
||||
gains.append(gain)
|
||||
|
||||
gains = np.array(gains)
|
||||
|
||||
if n_corr > 0:
|
||||
corr_gains = gains[corrections]
|
||||
helped = (corr_gains > 0).sum()
|
||||
print(f"\n Fusion corrections — MD gate gain vs mean gate:", flush=True)
|
||||
print(f" mean gain = {corr_gains.mean():+.4f} median = {np.median(corr_gains):+.4f}", flush=True)
|
||||
print(f" real MD helped {helped}/{n_corr} correction patients ({helped/n_corr:.0%})", flush=True)
|
||||
|
||||
if n_err > 0:
|
||||
err_gains = gains[errors]
|
||||
print(f"\n Fusion errors — MD gate gain vs mean gate:", flush=True)
|
||||
print(f" mean gain = {err_gains.mean():+.4f} median = {np.median(err_gains):+.4f}", flush=True)
|
||||
|
||||
# ---- save CSV ----
|
||||
import csv
|
||||
rows = []
|
||||
for idx in range(N):
|
||||
rows.append({
|
||||
"patient_idx": idx,
|
||||
"y_true": int(y_true[idx]),
|
||||
"pred_fused": int(pred_f[idx]),
|
||||
"pred_img": int(pred_i[idx]),
|
||||
"pred_md": int(pred_m[idx]),
|
||||
"conf_fused": float(pf[idx].max()),
|
||||
"conf_img": float(pi[idx].max()),
|
||||
"conf_md": float(pm[idx].max()),
|
||||
"is_correction": bool(corrections[idx]),
|
||||
"is_error": bool(errors[idx]),
|
||||
"md_gate_gain": float(gains[idx]),
|
||||
})
|
||||
csv_path = out_dir / "fusion_events.csv"
|
||||
with csv_path.open("w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
print(f" Saved → {csv_path}", flush=True)
|
||||
|
||||
# ---- chart ----
|
||||
fig, axes = plt.subplots(1, 2, figsize=(11, 4))
|
||||
|
||||
categories = ["corrections\n(both wrong→fused right)", "errors\n(both right→fused wrong)"]
|
||||
counts = [int(n_corr), int(n_err)]
|
||||
axes[0].bar(categories, counts, color=["#e05c5c", "#5c9ee0"], width=0.5)
|
||||
axes[0].set_ylabel("Count")
|
||||
axes[0].set_title(f"Fusion Events (N={N})")
|
||||
for i, v in enumerate(counts):
|
||||
axes[0].text(i, v + 0.1, str(v), ha="center", fontsize=11)
|
||||
|
||||
if n_corr > 0:
|
||||
axes[1].hist(gains[corrections], bins=10, alpha=0.7, color="#e05c5c", label=f"corrections (n={n_corr})")
|
||||
if n_err > 0:
|
||||
axes[1].hist(gains[errors], bins=10, alpha=0.7, color="#5c9ee0", label=f"errors (n={n_err})")
|
||||
axes[1].axvline(0, color="black", linewidth=0.8)
|
||||
axes[1].set_xlabel("MD gate gain vs mean gate\n(logit[true class]: real − mean)")
|
||||
axes[1].set_title("Does real MD help the fused prediction?")
|
||||
axes[1].legend(fontsize=9)
|
||||
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_dir / "fusion_events.png", dpi=150)
|
||||
plt.close(fig)
|
||||
print(f" Saved → {out_dir / 'fusion_events.png'}", flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
fold_dir = args.fold_dir.resolve()
|
||||
@@ -639,6 +871,15 @@ def main():
|
||||
out_dir=out_dir,
|
||||
)
|
||||
|
||||
# ---- Phase 3 ----
|
||||
if not args.no_phase3:
|
||||
run_fusion_event_analysis(
|
||||
model=model,
|
||||
loader=loader,
|
||||
device=device,
|
||||
out_dir=out_dir,
|
||||
)
|
||||
|
||||
print("\n[explain_fold] Done.", flush=True)
|
||||
|
||||
|
||||
|
||||
+281
@@ -0,0 +1,281 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Per-class ROC curves for v2 HyperTower runs.
|
||||
|
||||
Plots multiple runs as separate lines on the same axes — one figure per class
|
||||
(multiclass) or one figure total (binary).
|
||||
|
||||
v2 directory layout
|
||||
-------------------
|
||||
analysis_data/{run_name}/{eval_mode}/{tower_mode}/
|
||||
fold0/ y_true.npy probs_fused.npy | probs_bilat.npy | probs_classic.npy | probs_fused_head.npy
|
||||
fold1/ ...
|
||||
|
||||
Usage examples
|
||||
--------------
|
||||
# Compare UNet ensemble vs bilateral vs fused head (binary)
|
||||
python scripts/output_analysis/visualizations/aggregate_roc_perclass_all_models_v2.py \\
|
||||
--mode binary --tag unet_binary_comparison \\
|
||||
--runs \\
|
||||
analysis_data/v2_modes_full_40ep_5fold_roi_unet_perimage_refugebuild_holdout/binary/ensemble:"UNet Ensemble" \\
|
||||
analysis_data/v2_modes_full_40ep_5fold_roi_unet_perimage_refugebuild_holdout/binary/bilateral:"UNet Bilateral" \\
|
||||
analysis_data/v2_ensemble_fused_binary_unet_40ep_5fold_v1/binary/ensemble:"UNet Fused Head"
|
||||
|
||||
Each --runs entry is <path>:<label> where <path> points directly to the
|
||||
{eval_mode}/{tower_mode} subdirectory and <label> is shown in the legend.
|
||||
|
||||
Outputs (written to --output-dir, default: analysis_data/roc_plots/)
|
||||
{tag}_binary_roc.png (binary mode)
|
||||
{tag}_class{k}_roc.png (multiclass mode, one file per class)
|
||||
{tag}_roc_summary.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.metrics import roc_curve, auc as sk_auc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Probs filename auto-detection priority per tower mode
|
||||
# ---------------------------------------------------------------------------
|
||||
_PROBS_PRIORITY: dict[str, list[str]] = {
|
||||
"ensemble": ["probs_fused_head", "probs_fused"],
|
||||
"bilateral": ["probs_bilat"],
|
||||
"single": ["probs_classic"],
|
||||
"classic": ["probs_classic"],
|
||||
}
|
||||
_ALL_PROBS = ["probs_fused_head", "probs_fused", "probs_bilat", "probs_classic"]
|
||||
|
||||
|
||||
def _detect_probs_stem(fold_dir: Path, tower_mode: str | None) -> str | None:
|
||||
priority = _PROBS_PRIORITY.get(tower_mode, _ALL_PROBS) if tower_mode else _ALL_PROBS
|
||||
for stem in priority:
|
||||
if (fold_dir / f"{stem}.npy").exists():
|
||||
return stem
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fold discovery and loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def find_fold_dirs(mode_dir: Path) -> list[Path]:
|
||||
return sorted(
|
||||
[p for p in mode_dir.iterdir() if p.is_dir() and p.name.startswith("fold")],
|
||||
key=lambda p: int(p.name.replace("fold", "")),
|
||||
)
|
||||
|
||||
|
||||
def load_fold(fold_dir: Path, probs_stem: str) -> tuple[np.ndarray, np.ndarray] | None:
|
||||
y_path = fold_dir / "y_true.npy"
|
||||
p_path = fold_dir / f"{probs_stem}.npy"
|
||||
if not y_path.exists() or not p_path.exists():
|
||||
return None
|
||||
return np.load(y_path), np.load(p_path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-class ROC helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def per_class_roc(y: np.ndarray, p: np.ndarray) -> dict[int, tuple]:
|
||||
K = p.shape[1]
|
||||
out: dict[int, tuple] = {}
|
||||
for k in range(K):
|
||||
yb = (y == k).astype(np.uint8)
|
||||
if yb.sum() == 0 or yb.sum() == len(yb):
|
||||
continue
|
||||
fpr, tpr, _ = roc_curve(yb, p[:, k])
|
||||
out[k] = (fpr, tpr, sk_auc(fpr, tpr))
|
||||
return out
|
||||
|
||||
|
||||
def build_mean_curve(mode_dir: Path, tower_mode: str | None, mode: str) -> dict | None:
|
||||
fold_dirs = find_fold_dirs(mode_dir)
|
||||
if not fold_dirs:
|
||||
return None
|
||||
|
||||
probs_stem: str | None = None
|
||||
for fd in fold_dirs:
|
||||
probs_stem = _detect_probs_stem(fd, tower_mode)
|
||||
if probs_stem:
|
||||
break
|
||||
if probs_stem is None:
|
||||
return None
|
||||
|
||||
grid = np.linspace(0, 1, 501)
|
||||
per_fold: list[dict] = []
|
||||
for fd in fold_dirs:
|
||||
result = load_fold(fd, probs_stem)
|
||||
if result is None:
|
||||
continue
|
||||
y, p = result
|
||||
if mode == "binary":
|
||||
mask = np.isin(y, [0, 1])
|
||||
y, p = y[mask], p[mask]
|
||||
if p.shape[1] > 2:
|
||||
p = p[:, :2]
|
||||
per_fold.append(per_class_roc(y, p))
|
||||
|
||||
if not per_fold:
|
||||
return None
|
||||
|
||||
K = max(max(d.keys()) for d in per_fold) + 1
|
||||
class_curves: dict[int, dict] = {}
|
||||
for k in range(K):
|
||||
tprs, aucs = [], []
|
||||
for d in per_fold:
|
||||
if k not in d:
|
||||
continue
|
||||
fpr, tpr, a = d[k]
|
||||
tprs.append(np.interp(grid, fpr, tpr))
|
||||
aucs.append(a)
|
||||
if not tprs:
|
||||
continue
|
||||
tprs_arr = np.vstack(tprs)
|
||||
class_curves[k] = {
|
||||
"fpr": grid,
|
||||
"tpr_mean": tprs_arr.mean(axis=0),
|
||||
"tpr_std": tprs_arr.std(axis=0),
|
||||
"auc_mean": float(np.nanmean(aucs)),
|
||||
"auc_std": float(np.nanstd(aucs)),
|
||||
}
|
||||
|
||||
return {"probs_stem": probs_stem, "class_curves": class_curves}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def parse_run_entry(entry: str) -> tuple[Path, str]:
|
||||
"""Parse path:label or path (label defaults to last two dir components)."""
|
||||
if ":" in entry:
|
||||
raw_path, label = entry.rsplit(":", 1)
|
||||
else:
|
||||
raw_path = entry
|
||||
p = Path(entry)
|
||||
label = f"{p.parent.name}/{p.name}"
|
||||
return Path(raw_path), label
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(
|
||||
description="Per-class ROC curves comparing multiple v2 HyperTower runs.",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=__doc__,
|
||||
)
|
||||
ap.add_argument(
|
||||
"--runs", nargs="+", required=True, metavar="PATH[:LABEL]",
|
||||
help=(
|
||||
"Mode-level directories to compare, each optionally followed by :label. "
|
||||
"Path should point to the {eval_mode}/{tower_mode} subdirectory."
|
||||
),
|
||||
)
|
||||
ap.add_argument("--mode", required=True, choices=["binary", "multiclass"])
|
||||
ap.add_argument("--tag", required=True, help="Output filename prefix.")
|
||||
ap.add_argument(
|
||||
"--output-dir", default="analysis_data/roc_plots",
|
||||
help="Directory for PNG and JSON output (default: analysis_data/roc_plots).",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--class-names", nargs="*", default=["Healthy", "Glaucoma", "Suspect"],
|
||||
)
|
||||
ap.add_argument("--shade", action="store_true", help="Shade ±1 SD bands.")
|
||||
args = ap.parse_args()
|
||||
|
||||
out_dir = Path(args.output_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
per_model: list[tuple[str, dict]] = []
|
||||
for entry in args.runs:
|
||||
mode_dir, label = parse_run_entry(entry)
|
||||
if not mode_dir.exists():
|
||||
print(f" WARNING: {mode_dir} not found — skipping.")
|
||||
continue
|
||||
tower_mode = mode_dir.name
|
||||
result = build_mean_curve(mode_dir, tower_mode, args.mode)
|
||||
if result is None:
|
||||
print(f" WARNING: no usable folds in {mode_dir} — skipping.")
|
||||
continue
|
||||
auc_str = " ".join(
|
||||
f"class{k} AUC={v['auc_mean']:.3f}±{v['auc_std']:.3f}"
|
||||
for k, v in result["class_curves"].items()
|
||||
)
|
||||
print(f" {label} [{result['probs_stem']}] {auc_str}")
|
||||
per_model.append((label, result["class_curves"]))
|
||||
|
||||
if not per_model:
|
||||
raise SystemExit("No usable runs — nothing to plot.")
|
||||
|
||||
if args.mode == "binary":
|
||||
classes_to_plot = [1]
|
||||
out_names = [f"{args.tag}_binary_roc.png"]
|
||||
titles = ["Binary — Glaucoma (positive class)"]
|
||||
else:
|
||||
max_k = max(max(curves.keys()) for _, curves in per_model)
|
||||
classes_to_plot = list(range(min(3, max_k + 1)))
|
||||
out_names = [f"{args.tag}_class{k}_roc.png" for k in classes_to_plot]
|
||||
titles = [
|
||||
f"Multiclass OVR — "
|
||||
f"{args.class_names[k] if k < len(args.class_names) else f'class {k}'}"
|
||||
for k in classes_to_plot
|
||||
]
|
||||
|
||||
out_json: dict = {"tag": args.tag, "mode": args.mode, "figures": []}
|
||||
|
||||
for k, out_name, title in zip(classes_to_plot, out_names, titles):
|
||||
fig, ax = plt.subplots(figsize=(9, 7))
|
||||
ax.plot([0, 1], [0, 1], linestyle="--", linewidth=1, color="grey")
|
||||
ax.set_xlabel("False Positive Rate")
|
||||
ax.set_ylabel("True Positive Rate")
|
||||
ax.set_title(f"{title}\n{args.tag}")
|
||||
|
||||
entries = []
|
||||
for label, curves in per_model:
|
||||
if k not in curves:
|
||||
continue
|
||||
c = curves[k]
|
||||
ax.plot(
|
||||
c["fpr"], c["tpr_mean"], linewidth=2,
|
||||
label=f"{label} (AUC {c['auc_mean']:.3f} ± {c['auc_std']:.3f})",
|
||||
)
|
||||
if args.shade:
|
||||
ax.fill_between(
|
||||
c["fpr"],
|
||||
np.clip(c["tpr_mean"] - c["tpr_std"], 0, 1),
|
||||
np.clip(c["tpr_mean"] + c["tpr_std"], 0, 1),
|
||||
alpha=0.10,
|
||||
)
|
||||
entries.append({
|
||||
"label": label,
|
||||
"auc_mean": c["auc_mean"],
|
||||
"auc_std": c["auc_std"],
|
||||
})
|
||||
|
||||
ax.legend(loc="lower right")
|
||||
fig.tight_layout()
|
||||
out_path = out_dir / out_name
|
||||
fig.savefig(out_path, dpi=160)
|
||||
plt.close(fig)
|
||||
print(f" Saved: {out_path}")
|
||||
|
||||
out_json["figures"].append({
|
||||
"class_index": k,
|
||||
"output_png": str(out_path),
|
||||
"models": entries,
|
||||
})
|
||||
|
||||
summary_path = out_dir / f"{args.tag}_roc_summary.json"
|
||||
summary_path.write_text(json.dumps(out_json, indent=2))
|
||||
print(f" Summary: {summary_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,286 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Per-fold and mean OVR ROC plots for a single v2 HyperTower run.
|
||||
|
||||
Reads the saved .npy artifacts from a completed run and produces:
|
||||
- One per-fold ROC figure per class (all folds as individual lines)
|
||||
- One mean ± SD OVR ROC figure (all classes on the same axes)
|
||||
|
||||
Outputs are written to {mode_dir}/plots/.
|
||||
|
||||
v2 directory layout expected
|
||||
-----------------------------
|
||||
{run_dir}/{eval_mode}/{tower_mode}/
|
||||
fold0/ y_true.npy probs_fused.npy | probs_bilat.npy | probs_classic.npy | probs_fused_head.npy
|
||||
fold1/ ...
|
||||
|
||||
Usage examples
|
||||
--------------
|
||||
# Ensemble binary run
|
||||
python scripts/output_analysis/visualizations/plot_run_roc_v2.py \\
|
||||
--run-dir analysis_data/v2_modes_full_40ep_5fold_roi_unet_perimage_refugebuild_holdout \\
|
||||
--eval-mode binary --tower-mode ensemble
|
||||
|
||||
# Bilateral multiclass run
|
||||
python scripts/output_analysis/visualizations/plot_run_roc_v2.py \\
|
||||
--run-dir analysis_data/v2_modes_full_40ep_5fold_roi_unet_perimage_refugebuild_holdout \\
|
||||
--eval-mode multiclass --tower-mode bilateral
|
||||
|
||||
# Fused head — explicitly select probs file
|
||||
python scripts/output_analysis/visualizations/plot_run_roc_v2.py \\
|
||||
--run-dir analysis_data/v2_ensemble_fused_binary_unet_40ep_5fold_v1 \\
|
||||
--eval-mode binary --tower-mode ensemble --probs probs_fused_head
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.metrics import roc_curve, auc as sk_auc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Probs auto-detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PROBS_PRIORITY: dict[str, list[str]] = {
|
||||
"ensemble": ["probs_fused_head", "probs_fused"],
|
||||
"bilateral": ["probs_bilat"],
|
||||
"single": ["probs_classic"],
|
||||
"classic": ["probs_classic"],
|
||||
}
|
||||
_ALL_PROBS = ["probs_fused_head", "probs_fused", "probs_bilat", "probs_classic"]
|
||||
|
||||
|
||||
def detect_probs_stem(fold_dir: Path, tower_mode: str | None) -> str | None:
|
||||
priority = _PROBS_PRIORITY.get(tower_mode, _ALL_PROBS) if tower_mode else _ALL_PROBS
|
||||
for stem in priority:
|
||||
if (fold_dir / f"{stem}.npy").exists():
|
||||
return stem
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def find_fold_dirs(mode_dir: Path) -> list[Path]:
|
||||
return sorted(
|
||||
[p for p in mode_dir.iterdir() if p.is_dir() and p.name.startswith("fold")],
|
||||
key=lambda p: int(p.name.replace("fold", "")),
|
||||
)
|
||||
|
||||
|
||||
def _find_y_true(fold_dir: Path) -> Path | None:
|
||||
"""
|
||||
Return path to y_true.npy for this fold. If missing from fold_dir
|
||||
(can happen for bilateral-only old runs), fall back to the same fold
|
||||
index under sibling tower-mode directories (ensemble → single → classic).
|
||||
Labels are shared across tower modes within the same fold.
|
||||
"""
|
||||
local = fold_dir / "y_true.npy"
|
||||
if local.exists():
|
||||
return local
|
||||
fold_name = fold_dir.name # e.g. "fold0"
|
||||
tower_dir = fold_dir.parent # e.g. .../binary/bilateral
|
||||
eval_dir = tower_dir.parent # e.g. .../binary
|
||||
for fallback in ("ensemble", "single", "classic"):
|
||||
candidate = eval_dir / fallback / fold_name / "y_true.npy"
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def load_fold(
|
||||
fold_dir: Path,
|
||||
probs_stem: str,
|
||||
eval_mode: str,
|
||||
) -> tuple[np.ndarray, np.ndarray] | None:
|
||||
y_path = _find_y_true(fold_dir)
|
||||
p_path = fold_dir / f"{probs_stem}.npy"
|
||||
if y_path is None or not p_path.exists():
|
||||
return None
|
||||
y = np.load(y_path)
|
||||
p = np.load(p_path)
|
||||
if eval_mode == "binary":
|
||||
mask = np.isin(y, [0, 1])
|
||||
y, p = y[mask], p[mask]
|
||||
if p.shape[1] > 2:
|
||||
p = p[:, :2]
|
||||
return y, p
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ROC helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def per_class_roc(y: np.ndarray, p: np.ndarray) -> dict[int, dict]:
|
||||
"""OVR ROC for each class. Returns {k: {fpr, tpr, auc}}."""
|
||||
out: dict[int, dict] = {}
|
||||
for k in range(p.shape[1]):
|
||||
yb = (y == k).astype(np.uint8)
|
||||
if yb.sum() == 0 or yb.sum() == len(yb):
|
||||
continue
|
||||
fpr, tpr, _ = roc_curve(yb, p[:, k])
|
||||
out[k] = {"fpr": fpr, "tpr": tpr, "auc": sk_auc(fpr, tpr)}
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plotting
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def plot_perfold(
|
||||
per_fold: list[tuple[int, dict]],
|
||||
out_dir: Path,
|
||||
class_names: list[str],
|
||||
probs_stem: str,
|
||||
eval_mode: str,
|
||||
) -> None:
|
||||
"""One figure per class: each fold as a separate line."""
|
||||
all_classes = sorted({k for _, curves in per_fold for k in curves})
|
||||
if eval_mode == "binary":
|
||||
all_classes = [k for k in all_classes if k == 1]
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for k in all_classes:
|
||||
cname = class_names[k] if k < len(class_names) else f"class_{k}"
|
||||
fig, ax = plt.subplots(figsize=(9, 7))
|
||||
ax.plot([0, 1], [0, 1], linestyle="--", linewidth=1, color="grey")
|
||||
for fold_idx, curves in per_fold:
|
||||
if k not in curves:
|
||||
continue
|
||||
c = curves[k]
|
||||
auc_val = c["auc"]
|
||||
ax.plot(c["fpr"], c["tpr"], linewidth=1.5,
|
||||
label=f"Fold {fold_idx} (AUC {auc_val:.3f})")
|
||||
ax.set_xlabel("False Positive Rate")
|
||||
ax.set_ylabel("True Positive Rate")
|
||||
ax.set_title(f"Per-fold ROC — {cname} [{probs_stem}]")
|
||||
ax.legend(loc="lower right")
|
||||
fig.tight_layout()
|
||||
safe = cname.replace(" ", "_")
|
||||
fig.savefig(out_dir / f"roc_{probs_stem}_{safe}_perfold.png", dpi=160)
|
||||
plt.close(fig)
|
||||
print(f" Saved per-fold ROC ({cname})")
|
||||
|
||||
|
||||
def plot_mean_ovr(
|
||||
per_fold: list[tuple[int, dict]],
|
||||
out_dir: Path,
|
||||
class_names: list[str],
|
||||
probs_stem: str,
|
||||
eval_mode: str,
|
||||
) -> None:
|
||||
"""Mean ± SD OVR ROC — all classes on one figure."""
|
||||
all_classes = sorted({k for _, curves in per_fold for k in curves})
|
||||
if eval_mode == "binary":
|
||||
all_classes = [k for k in all_classes if k == 1]
|
||||
grid = np.linspace(0, 1, 501)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
fig, ax = plt.subplots(figsize=(9, 7))
|
||||
ax.plot([0, 1], [0, 1], linestyle="--", linewidth=1, color="grey")
|
||||
for k in all_classes:
|
||||
cname = class_names[k] if k < len(class_names) else f"class_{k}"
|
||||
tprs, aucs = [], []
|
||||
for _, curves in per_fold:
|
||||
if k not in curves:
|
||||
continue
|
||||
c = curves[k]
|
||||
tprs.append(np.interp(grid, c["fpr"], c["tpr"]))
|
||||
aucs.append(c["auc"])
|
||||
if not tprs:
|
||||
continue
|
||||
tprs_arr = np.vstack(tprs)
|
||||
mean = tprs_arr.mean(axis=0)
|
||||
std = tprs_arr.std(axis=0)
|
||||
label = f"{cname} (AUC {np.nanmean(aucs):.3f} ± {np.nanstd(aucs):.3f})"
|
||||
line, = ax.plot(grid, mean, linewidth=2, label=label)
|
||||
ax.fill_between(grid,
|
||||
np.clip(mean - std, 0, 1),
|
||||
np.clip(mean + std, 0, 1),
|
||||
alpha=0.15, color=line.get_color())
|
||||
ax.set_xlabel("False Positive Rate")
|
||||
ax.set_ylabel("True Positive Rate")
|
||||
ax.set_title(f"Mean OVR ROC (± 1 SD) [{probs_stem}]")
|
||||
ax.legend(loc="lower right")
|
||||
fig.tight_layout()
|
||||
out_path = out_dir / f"roc_{probs_stem}_mean_ovr.png"
|
||||
fig.savefig(out_path, dpi=160)
|
||||
plt.close(fig)
|
||||
print(f" Saved mean OVR ROC: {out_path}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(
|
||||
description="Per-fold and mean OVR ROC plots for a single v2 run.",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=__doc__,
|
||||
)
|
||||
ap.add_argument("--run-dir", required=True, type=Path,
|
||||
help="Top-level run directory (e.g. analysis_data/v2_my_run).")
|
||||
ap.add_argument("--eval-mode", required=True, choices=["binary", "multiclass"])
|
||||
ap.add_argument("--tower-mode", required=True,
|
||||
choices=["single", "classic", "ensemble", "bilateral"],
|
||||
help="Tower mode subdirectory to read from.")
|
||||
ap.add_argument("--probs", default=None,
|
||||
help="Probs file stem to use (e.g. probs_fused, probs_bilat, "
|
||||
"probs_fused_head). Auto-detected if omitted.")
|
||||
ap.add_argument("--class-names", nargs="*",
|
||||
default=["Healthy", "Glaucoma", "Suspect"])
|
||||
args = ap.parse_args()
|
||||
|
||||
mode_dir = args.run_dir / args.eval_mode / args.tower_mode
|
||||
if not mode_dir.exists():
|
||||
raise SystemExit(f"Directory not found: {mode_dir}")
|
||||
|
||||
fold_dirs = find_fold_dirs(mode_dir)
|
||||
if not fold_dirs:
|
||||
raise SystemExit(f"No fold subdirectories found in {mode_dir}")
|
||||
|
||||
# Determine probs stem
|
||||
probs_stem = args.probs
|
||||
if probs_stem is None:
|
||||
for fd in fold_dirs:
|
||||
probs_stem = detect_probs_stem(fd, args.tower_mode)
|
||||
if probs_stem:
|
||||
break
|
||||
if probs_stem is None:
|
||||
raise SystemExit(f"Could not detect a probs file in {mode_dir}/fold*/")
|
||||
print(f"Using probs: {probs_stem}.npy")
|
||||
|
||||
# Load all folds
|
||||
per_fold: list[tuple[int, dict]] = []
|
||||
for fd in fold_dirs:
|
||||
fold_idx = int(fd.name.replace("fold", ""))
|
||||
result = load_fold(fd, probs_stem, args.eval_mode)
|
||||
if result is None:
|
||||
print(f" [skip] fold {fold_idx}: missing y_true or {probs_stem}.npy")
|
||||
continue
|
||||
y, p = result
|
||||
curves = per_class_roc(y, p)
|
||||
per_fold.append((fold_idx, curves))
|
||||
auc_str = " ".join(
|
||||
f"class{k}={v['auc']:.3f}" for k, v in curves.items()
|
||||
)
|
||||
print(f" fold {fold_idx}: {auc_str}")
|
||||
|
||||
if not per_fold:
|
||||
raise SystemExit("No usable folds — nothing to plot.")
|
||||
|
||||
out_dir = mode_dir / "plots"
|
||||
plot_perfold(per_fold, out_dir, args.class_names, probs_stem, args.eval_mode)
|
||||
plot_mean_ovr(per_fold, out_dir, args.class_names, probs_stem, args.eval_mode)
|
||||
print(f"\nPlots written to {out_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -19,7 +19,7 @@ from types import SimpleNamespace
|
||||
import matplotlib
|
||||
import sys
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
matplotlib.use("Agg")
|
||||
@@ -70,10 +70,34 @@ def load_summary(run_dir: Path) -> dict:
|
||||
return json.load(fh)
|
||||
|
||||
|
||||
def resolve_data_dir(raw_dir: str) -> str:
|
||||
"""
|
||||
Resolve dataset paths saved in legacy cli_args.json.
|
||||
|
||||
Older runs often store "ClinicalData"/"FundusImages" relative to a
|
||||
dataset root, while current repo layout uses "Papila/<dir>".
|
||||
"""
|
||||
p = Path(raw_dir)
|
||||
if p.exists():
|
||||
return str(p)
|
||||
|
||||
candidates = [
|
||||
REPO_ROOT / p,
|
||||
REPO_ROOT / "Papila" / p,
|
||||
]
|
||||
for cand in candidates:
|
||||
if cand.exists():
|
||||
return str(cand)
|
||||
|
||||
return str(p)
|
||||
|
||||
|
||||
def prepare_clinical(cli_args: dict, run_dir: Path) -> tuple:
|
||||
image_dir = resolve_data_dir(cli_args["image_dir"])
|
||||
clinical_dir = resolve_data_dir(cli_args["clinical_dir"])
|
||||
clinical = build_papila_clinical(
|
||||
cli_args["image_dir"],
|
||||
cli_args["clinical_dir"],
|
||||
image_dir,
|
||||
clinical_dir,
|
||||
cli_args["label_col"],
|
||||
cli_args["cat_cols"],
|
||||
n_splits=cli_args["n_splits"],
|
||||
@@ -103,10 +127,12 @@ def prepare_clinical(cli_args: dict, run_dir: Path) -> tuple:
|
||||
|
||||
|
||||
def build_ht_args(cli_args: dict, fold: int, run_dir: Path, models_dir: Path, holdout_df):
|
||||
image_dir = resolve_data_dir(cli_args["image_dir"])
|
||||
clinical_dir = resolve_data_dir(cli_args["clinical_dir"])
|
||||
# Copy of the training-time namespace so HyperTower can be re-instantiated.
|
||||
return SimpleNamespace(
|
||||
image_dir=cli_args["image_dir"],
|
||||
clinical_dir=cli_args["clinical_dir"],
|
||||
image_dir=image_dir,
|
||||
clinical_dir=clinical_dir,
|
||||
label_col=cli_args["label_col"],
|
||||
cat_cols=cli_args["cat_cols"],
|
||||
batch_size=cli_args["batch_size"],
|
||||
@@ -253,6 +279,14 @@ def choose_head_probs(head: str, probs_f, probs_i, probs_m):
|
||||
return probs_i
|
||||
|
||||
|
||||
def legacy_probs_suffix(head: str) -> str:
|
||||
if head == "image":
|
||||
return "img"
|
||||
if head == "metadata":
|
||||
return "md"
|
||||
return "fused"
|
||||
|
||||
|
||||
def ensure_binary_slice(y_true, *arrays):
|
||||
mask = np.isin(y_true, [0, 1])
|
||||
filtered = [y_true[mask]]
|
||||
@@ -264,8 +298,17 @@ def ensure_binary_slice(y_true, *arrays):
|
||||
return filtered
|
||||
|
||||
|
||||
def plot_overlays(per_fold_curves, out_dir: Path, class_names: list[str], head: str, suffix: str = ""):
|
||||
def plot_overlays(
|
||||
per_fold_curves,
|
||||
out_dir: Path,
|
||||
class_names: list[str],
|
||||
head: str,
|
||||
eval_mode: str,
|
||||
suffix: str = "",
|
||||
):
|
||||
keys = sorted({k for _, curves in per_fold_curves for k in curves.keys()})
|
||||
if eval_mode == "binary":
|
||||
keys = [k for k in keys if k == 1]
|
||||
if not keys:
|
||||
return
|
||||
name_map = {k: (class_names[k] if k < len(class_names) else f"class_{k}") for k in keys}
|
||||
@@ -293,8 +336,17 @@ def plot_overlays(per_fold_curves, out_dir: Path, class_names: list[str], head:
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def plot_mean_sd(per_fold_curves, out_dir: Path, class_names: list[str], head: str, suffix: str = ""):
|
||||
def plot_mean_sd(
|
||||
per_fold_curves,
|
||||
out_dir: Path,
|
||||
class_names: list[str],
|
||||
head: str,
|
||||
eval_mode: str,
|
||||
suffix: str = "",
|
||||
):
|
||||
keys = sorted({k for _, curves in per_fold_curves for k in curves.keys()})
|
||||
if eval_mode == "binary":
|
||||
keys = [k for k in keys if k == 1]
|
||||
if not keys:
|
||||
return
|
||||
grid = np.linspace(0, 1, 501)
|
||||
@@ -363,10 +415,43 @@ def main():
|
||||
print(f"[skip] Fold {fold_idx}: no best_epoch recorded.")
|
||||
continue
|
||||
|
||||
# Prefer saved fold arrays when available. This avoids reconstructing
|
||||
# HyperTower for legacy runs whose external weight paths no longer exist.
|
||||
base = run_dir / f"fold{fold_idx}{file_suffix}"
|
||||
y_path = Path(f"{base}_y_true.npy")
|
||||
p_path = Path(f"{base}_probs_{legacy_probs_suffix(head)}.npy")
|
||||
if y_path.exists() and p_path.exists():
|
||||
y_true = np.load(y_path)
|
||||
head_probs = np.load(p_path)
|
||||
if cli_args["eval_mode"] == "binary" and head_probs.shape[1] >= 2:
|
||||
head_probs = head_probs[:, :2]
|
||||
curves = compute_per_class_curves(y_true, head_probs)
|
||||
per_fold_curves.append((fold_idx, curves))
|
||||
try:
|
||||
if head_probs.shape[1] > 2:
|
||||
fold_auc = roc_auc_score(y_true, head_probs, multi_class="ovr", average="macro")
|
||||
else:
|
||||
target_scores = head_probs[:, 1] if head_probs.shape[1] > 1 else head_probs[:, 0]
|
||||
fold_auc = roc_auc_score(y_true, target_scores)
|
||||
fold_aucs.append(fold_auc)
|
||||
print(
|
||||
f"[info] Fold {fold_idx}: using saved arrays "
|
||||
f"({y_path.name}, {p_path.name}), AUC={fold_auc:.4f}"
|
||||
)
|
||||
except Exception:
|
||||
print(
|
||||
f"[warning] Fold {fold_idx}: using saved arrays "
|
||||
f"({y_path.name}, {p_path.name}) but AUC failed."
|
||||
)
|
||||
continue
|
||||
|
||||
fold_models_dir = base_models_dir / f"fold{fold_idx}"
|
||||
best_checkpoint = fold_models_dir / "model_best.pt"
|
||||
if not best_checkpoint.exists():
|
||||
print(f"[warning] Fold {fold_idx}: missing model_best.pt at {best_checkpoint}")
|
||||
print(
|
||||
f"[warning] Fold {fold_idx}: missing model_best.pt at {best_checkpoint} "
|
||||
f"and missing fallback arrays {y_path.name}/{p_path.name}"
|
||||
)
|
||||
continue
|
||||
|
||||
ht_args = build_ht_args(cli_args, fold_idx, run_dir, fold_models_dir, holdout_df)
|
||||
@@ -435,8 +520,8 @@ def main():
|
||||
raise SystemExit("No folds processed; nothing to plot.")
|
||||
|
||||
plots_dir = run_dir / "plots"
|
||||
plot_overlays(per_fold_curves, plots_dir, class_names, head, file_suffix)
|
||||
plot_mean_sd(per_fold_curves, plots_dir, class_names, head, file_suffix)
|
||||
plot_overlays(per_fold_curves, plots_dir, class_names, head, cli_args["eval_mode"], file_suffix)
|
||||
plot_mean_sd(per_fold_curves, plots_dir, class_names, head, cli_args["eval_mode"], file_suffix)
|
||||
|
||||
if fold_aucs:
|
||||
print(f"[info] {head} head mean AUC across folds: {np.mean(fold_aucs):.4f} ± {np.std(fold_aucs):.4f}")
|
||||
|
||||
Reference in New Issue
Block a user