pre-refactor 041426
This commit is contained in:
rpotter6298
@@ -0,0 +1,809 @@
|
||||
"""
|
||||
Phase 1–5 analysis — ablation plots + pairwise Wilcoxon tests.
|
||||
|
||||
Usage:
|
||||
python -m v3.scripts.output_analysis.analyze_phases # all phases
|
||||
python -m v3.scripts.output_analysis.analyze_phases --phase 3
|
||||
python -m v3.scripts.output_analysis.analyze_phases --out figures/
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from scipy.stats import wilcoxon
|
||||
from sklearn.metrics import roc_auc_score
|
||||
from statsmodels.stats.multitest import multipletests
|
||||
|
||||
RESULTS_ROOT = Path(__file__).resolve().parents[3] / "v3" / "results"
|
||||
|
||||
# Shared style constants
|
||||
C_BASELINE = "#dd8452"
|
||||
C_OTHER = "#4c72b0"
|
||||
C_MEDIAN = "#c44e52"
|
||||
FSIZE = 10
|
||||
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def load_rep_aucs(run_path: Path, test_key: str = "classic_test") -> np.ndarray:
|
||||
"""Return array of per-rep AUC means for a run directory."""
|
||||
aucs = []
|
||||
for rep_dir in sorted(run_path.glob("rep*")):
|
||||
for summary in rep_dir.rglob("summary.json"):
|
||||
txt = summary.read_text().strip()
|
||||
if not txt:
|
||||
continue
|
||||
d = json.loads(txt)
|
||||
auc = d.get("mode_summary", {}).get(test_key, {}).get("auc_mean")
|
||||
if auc is not None:
|
||||
aucs.append(auc)
|
||||
break # one summary per rep
|
||||
return np.array(aucs)
|
||||
|
||||
|
||||
def wilcoxon_p(a: np.ndarray, b: np.ndarray) -> float:
|
||||
"""Two-sided Wilcoxon signed-rank p-value; returns nan if underpowered."""
|
||||
diffs = a - b
|
||||
if np.all(diffs == 0) or len(diffs) < 5:
|
||||
return float("nan")
|
||||
try:
|
||||
return wilcoxon(diffs, alternative="two-sided").pvalue
|
||||
except Exception:
|
||||
return float("nan")
|
||||
|
||||
|
||||
def stars(p: float) -> str:
|
||||
if np.isnan(p): return ""
|
||||
if p < 0.001: return "***"
|
||||
if p < 0.01: return "**"
|
||||
if p < 0.05: return "*"
|
||||
return "ns"
|
||||
|
||||
|
||||
def paired_matrix(runs: list[str], aucs_dict: dict[str, np.ndarray],
|
||||
fdr: bool = True) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""Return (p_matrix, corrected_p_matrix) shape (n, n)."""
|
||||
n = len(runs)
|
||||
raw = np.full((n, n), np.nan)
|
||||
for i, a in enumerate(runs):
|
||||
for j, b in enumerate(runs):
|
||||
if i != j and a in aucs_dict and b in aucs_dict:
|
||||
ai, bi = aucs_dict[a], aucs_dict[b]
|
||||
min_n = min(len(ai), len(bi))
|
||||
if min_n >= 5:
|
||||
raw[i, j] = wilcoxon_p(ai[:min_n], bi[:min_n])
|
||||
if fdr:
|
||||
mask = ~np.isnan(raw)
|
||||
if mask.sum() > 0:
|
||||
flat = raw[mask]
|
||||
_, corrected, _, _ = multipletests(flat, method="fdr_bh")
|
||||
corr = raw.copy()
|
||||
corr[mask] = corrected
|
||||
return raw, corr
|
||||
return raw, raw.copy()
|
||||
|
||||
|
||||
def boxplot_panel(ax, data_list, labels, base_idx, title="", ylabel="Test AUC",
|
||||
base_aucs=None, all_aucs_by_label=None):
|
||||
"""Vertical box plot with p-value vs baseline under each tick label."""
|
||||
n = len(labels)
|
||||
x = np.arange(n)
|
||||
colors = [C_BASELINE if i == base_idx else C_OTHER for i in range(n)]
|
||||
|
||||
bp = ax.boxplot(data_list, vert=True, patch_artist=True, positions=x,
|
||||
widths=0.3, showfliers=True,
|
||||
flierprops=dict(marker="o", markersize=3, alpha=0.5),
|
||||
medianprops=dict(color=C_MEDIAN, linewidth=2))
|
||||
for patch, color in zip(bp["boxes"], colors):
|
||||
patch.set_facecolor(color)
|
||||
patch.set_alpha(0.8)
|
||||
|
||||
if base_aucs is not None:
|
||||
ax.axhline(np.median(base_aucs), color=C_BASELINE, linewidth=1,
|
||||
linestyle="--", alpha=0.5, label="Baseline median")
|
||||
ax.legend(fontsize=FSIZE - 1)
|
||||
|
||||
ax.set_xlim(-0.5, n - 0.5)
|
||||
ax.set_ylabel(ylabel, fontsize=FSIZE + 1)
|
||||
if title:
|
||||
ax.set_title(title, fontsize=FSIZE + 1, fontweight="bold")
|
||||
ax.grid(axis="y", alpha=0.3)
|
||||
|
||||
tick_labels = []
|
||||
for i, lbl in enumerate(labels):
|
||||
if i == base_idx or base_aucs is None:
|
||||
tick_labels.append(lbl)
|
||||
continue
|
||||
a = (all_aucs_by_label or {}).get(lbl, data_list[i])
|
||||
min_n = min(len(a), len(base_aucs))
|
||||
p = wilcoxon_p(a[:min_n], base_aucs[:min_n])
|
||||
p_str = f"p={p:.3f}" if not np.isnan(p) else "p=n/a"
|
||||
tick_labels.append(f"{lbl}\n{p_str}")
|
||||
|
||||
ax.set_xticks(x)
|
||||
ax.set_xticklabels(tick_labels, fontsize=FSIZE)
|
||||
|
||||
|
||||
def bar_plot(ax, labels, means, stds, baseline_idx, title, ylabel="AUC",
|
||||
baseline_aucs=None, all_aucs=None):
|
||||
"""Horizontal bar chart with baseline highlighted and p-value annotations."""
|
||||
n = len(labels)
|
||||
colors = ["#4c72b0" if i != baseline_idx else "#dd8452" for i in range(n)]
|
||||
y = np.arange(n)
|
||||
bars = ax.barh(y, means, xerr=stds, color=colors, alpha=0.85,
|
||||
height=0.6, capsize=3, error_kw=dict(linewidth=1))
|
||||
ax.set_yticks(y)
|
||||
ax.set_yticklabels(labels, fontsize=8)
|
||||
ax.set_xlabel(ylabel)
|
||||
ax.set_title(title, fontsize=10, fontweight="bold")
|
||||
ax.axvline(means[baseline_idx], color="#dd8452", linewidth=1, linestyle="--", alpha=0.6)
|
||||
|
||||
# Annotate with p-value stars vs baseline
|
||||
if baseline_aucs is not None and all_aucs is not None:
|
||||
x_max = max(means) + max(stds) + 0.005
|
||||
for i, lbl in enumerate(labels):
|
||||
if i == baseline_idx:
|
||||
continue
|
||||
a = all_aucs.get(lbl)
|
||||
if a is None:
|
||||
continue
|
||||
min_n = min(len(a), len(baseline_aucs))
|
||||
p = wilcoxon_p(a[:min_n], baseline_aucs[:min_n])
|
||||
s = stars(p)
|
||||
if s:
|
||||
ax.text(x_max, i, s, va="center", fontsize=7,
|
||||
color="black" if s != "ns" else "gray")
|
||||
|
||||
|
||||
def pairwise_heatmap(ax, runs, p_matrix, title):
|
||||
"""Lower-triangle heatmap of corrected p-values."""
|
||||
n = len(runs)
|
||||
display = np.full_like(p_matrix, np.nan)
|
||||
for i in range(n):
|
||||
for j in range(i):
|
||||
display[i, j] = p_matrix[i, j]
|
||||
|
||||
im = ax.imshow(display, vmin=0, vmax=0.1, cmap="RdYlGn_r", aspect="auto")
|
||||
ax.set_xticks(range(n))
|
||||
ax.set_yticks(range(n))
|
||||
ax.set_xticklabels(runs, rotation=45, ha="right", fontsize=7)
|
||||
ax.set_yticklabels(runs, fontsize=7)
|
||||
ax.set_title(title, fontsize=10, fontweight="bold")
|
||||
plt.colorbar(im, ax=ax, label="p-value (FDR)")
|
||||
|
||||
for i in range(n):
|
||||
for j in range(i):
|
||||
p = display[i, j]
|
||||
if not np.isnan(p):
|
||||
ax.text(j, i, f"{p:.2f}", ha="center", va="center",
|
||||
fontsize=6, color="white" if p < 0.05 else "black")
|
||||
|
||||
|
||||
# ── Phase 1 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
P1_CLF_ORDER = ["KNN", "Random Forest", "SVM", "Logistic Regression"]
|
||||
P1_CLF_LABELS = {"KNN": "KNN", "Random Forest": "RF", "SVM": "SVM", "Logistic Regression": "LR"}
|
||||
P1_TAGS = [
|
||||
("no_leakage", "baseline"),
|
||||
("hypertower_loader", "HT loader"),
|
||||
]
|
||||
P1_BASELINE_TAG = "no_leakage"
|
||||
P1_PAPER_AUC = {"KNN": 0.75, "Random Forest": 0.64, "SVM": 0.75, "Logistic Regression": 0.70}
|
||||
|
||||
P1_BACKBONES = ["densenet121", "vgg16", "mobilenet_v2", "inception_v3", "resnet50"]
|
||||
P1_BACKBONE_LABELS = {
|
||||
"densenet121": "DenseNet121",
|
||||
"vgg16": "VGG16",
|
||||
"mobilenet_v2": "MobileNetV2",
|
||||
"inception_v3": "InceptionV3",
|
||||
"resnet50": "ResNet50",
|
||||
}
|
||||
P1_PAPER_CNN = {
|
||||
"densenet121": (0.80, 0.05), "vgg16": (0.84, 0.02),
|
||||
"mobilenet_v2": (0.75, 0.06), "inception_v3": (0.78, 0.08), "resnet50": (0.78, 0.07),
|
||||
}
|
||||
|
||||
|
||||
def _load_p1_clf_aucs(phase1_dir: Path) -> dict:
|
||||
"""Load per-fold AUCs for each tag × classifier."""
|
||||
data = {}
|
||||
for tag, _ in P1_TAGS:
|
||||
data[tag] = {}
|
||||
for clf in P1_CLF_ORDER:
|
||||
fpath = phase1_dir / tag / clf / "fold_metrics.csv"
|
||||
if fpath.exists():
|
||||
data[tag][clf] = pd.read_csv(fpath)["auc"].tolist()
|
||||
return data
|
||||
|
||||
|
||||
def _load_p1_cnn_aucs(phase1_dir: Path) -> tuple[dict, dict]:
|
||||
cnn, ht = {}, {}
|
||||
for b in P1_BACKBONES:
|
||||
fpath = phase1_dir / f"cnn_{b}" / "fold_metrics.csv"
|
||||
cnn[b] = pd.read_csv(fpath)["auc"].tolist() if fpath.exists() else []
|
||||
aucs = []
|
||||
for fold in range(5):
|
||||
yp = phase1_dir / "imageonly_ht" / b / "binary" / "single" / f"fold{fold}" / "test_y_true.npy"
|
||||
pp = phase1_dir / "imageonly_ht" / b / "binary" / "single" / f"fold{fold}" / "test_probs_fused.npy"
|
||||
if yp.exists() and pp.exists():
|
||||
y, pr = np.load(yp), np.load(pp)
|
||||
if len(np.unique(y)) >= 2:
|
||||
aucs.append(float(roc_auc_score(y, pr[:, 1])))
|
||||
ht[b] = aucs
|
||||
return cnn, ht
|
||||
|
||||
|
||||
def analyze_phase1(out_dir: Path):
|
||||
print("\n=== Phase 1 ===")
|
||||
phase1_dir = RESULTS_ROOT / "phase1"
|
||||
|
||||
# ── Clinical classifiers ────────────────────────────────────────────────
|
||||
clf_data = _load_p1_clf_aucs(phase1_dir)
|
||||
n_clf = len(P1_CLF_ORDER)
|
||||
n_tags = len(P1_TAGS)
|
||||
group_w = 0.7
|
||||
box_w = group_w / n_tags * 0.85
|
||||
offsets = np.linspace(-group_w / 2 + box_w / 2, group_w / 2 - box_w / 2, n_tags)
|
||||
tag_colors = [C_BASELINE if t == P1_BASELINE_TAG else C_OTHER for t, _ in P1_TAGS]
|
||||
|
||||
fig1, ax1 = plt.subplots(figsize=(10, 5))
|
||||
fig1.suptitle("Phase 1 — Clinical-only classifiers: CV strategy comparison",
|
||||
fontsize=FSIZE + 2, fontweight="bold")
|
||||
|
||||
for ti, (tag, lbl) in enumerate(P1_TAGS):
|
||||
color = tag_colors[ti]
|
||||
first = True
|
||||
for ci, clf in enumerate(P1_CLF_ORDER):
|
||||
aucs = clf_data.get(tag, {}).get(clf, [])
|
||||
if not aucs:
|
||||
continue
|
||||
bp = ax1.boxplot(aucs, positions=[ci + offsets[ti]], widths=box_w,
|
||||
patch_artist=True, manage_ticks=False,
|
||||
boxprops=dict(facecolor=color, alpha=0.8),
|
||||
medianprops=dict(color=C_MEDIAN, linewidth=2),
|
||||
whiskerprops=dict(color=color, linewidth=1.2),
|
||||
capprops=dict(color=color, linewidth=1.2),
|
||||
flierprops=dict(marker="o", markersize=3, alpha=0.5))
|
||||
if first:
|
||||
bp["boxes"][0].set_label(lbl)
|
||||
first = False
|
||||
|
||||
for ci, clf in enumerate(P1_CLF_ORDER):
|
||||
if clf in P1_PAPER_AUC:
|
||||
ax1.hlines(P1_PAPER_AUC[clf], ci - group_w / 2, ci + group_w / 2,
|
||||
colors="black", linestyles=":", linewidths=1.5,
|
||||
label="PAPILA paper" if ci == 0 else "_nolegend_")
|
||||
|
||||
ax1.set_xticks(range(n_clf))
|
||||
ax1.set_xticklabels([P1_CLF_LABELS[c] for c in P1_CLF_ORDER], fontsize=FSIZE + 1)
|
||||
ax1.set_ylabel("Test AUC", fontsize=FSIZE + 1)
|
||||
ax1.set_ylim(0.45, 1.02)
|
||||
ax1.axhline(0.5, color="grey", linestyle="--", linewidth=0.8, alpha=0.4)
|
||||
ax1.grid(axis="y", alpha=0.3)
|
||||
ax1.legend(fontsize=FSIZE, loc="lower right", framealpha=0.9)
|
||||
fig1.tight_layout()
|
||||
p1 = out_dir / "phase1_clinical_classifiers.png"
|
||||
fig1.savefig(p1, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig1)
|
||||
print(f" Saved: {p1}")
|
||||
|
||||
# ── CNN backbones ───────────────────────────────────────────────────────
|
||||
cnn_data, ht_data = _load_p1_cnn_aucs(phase1_dir)
|
||||
n_b = len(P1_BACKBONES)
|
||||
offsets2 = [-group_w / 4, group_w / 4]
|
||||
method_colors = [C_OTHER, C_BASELINE]
|
||||
|
||||
fig2, ax2 = plt.subplots(figsize=(11, 5))
|
||||
fig2.suptitle("Phase 1 — CNN backbone: standalone vs HyperTower (image only)",
|
||||
fontsize=FSIZE + 2, fontweight="bold")
|
||||
|
||||
for bi, backbone in enumerate(P1_BACKBONES):
|
||||
for si, (lbl, data, color) in enumerate([
|
||||
("CNN standalone", cnn_data, method_colors[0]),
|
||||
("HyperTower", ht_data, method_colors[1]),
|
||||
]):
|
||||
aucs = data.get(backbone, [])
|
||||
if not aucs:
|
||||
continue
|
||||
bp = ax2.boxplot(aucs, positions=[bi + offsets2[si]], widths=box_w,
|
||||
patch_artist=True, manage_ticks=False,
|
||||
boxprops=dict(facecolor=color, alpha=0.8),
|
||||
medianprops=dict(color=C_MEDIAN, linewidth=2),
|
||||
whiskerprops=dict(color=color, linewidth=1.2),
|
||||
capprops=dict(color=color, linewidth=1.2),
|
||||
flierprops=dict(marker="o", markersize=3, alpha=0.5))
|
||||
if bi == 0:
|
||||
bp["boxes"][0].set_label(lbl)
|
||||
|
||||
if backbone in P1_PAPER_CNN:
|
||||
mean_p, _ = P1_PAPER_CNN[backbone]
|
||||
ax2.hlines(mean_p, bi - group_w / 2, bi + group_w / 2,
|
||||
colors="black", linestyles=":", linewidths=1.5,
|
||||
label="PAPILA paper" if bi == 0 else "_nolegend_")
|
||||
|
||||
ax2.set_xticks(range(n_b))
|
||||
ax2.set_xticklabels([P1_BACKBONE_LABELS[b] for b in P1_BACKBONES], fontsize=FSIZE)
|
||||
ax2.set_ylabel("Test AUC", fontsize=FSIZE + 1)
|
||||
ax2.set_ylim(0.45, 1.02)
|
||||
ax2.axhline(0.5, color="grey", linestyle="--", linewidth=0.8, alpha=0.4)
|
||||
ax2.grid(axis="y", alpha=0.3)
|
||||
ax2.legend(fontsize=FSIZE, loc="lower right", framealpha=0.9)
|
||||
fig2.tight_layout()
|
||||
p2 = out_dir / "phase1_cnn_backbones.png"
|
||||
fig2.savefig(p2, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig2)
|
||||
print(f" Saved: {p2}")
|
||||
|
||||
|
||||
# ── Phase 2 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
PHASE2_RUNS = [
|
||||
("imageonly_resnet50_leaky", "classic_test", "leaky CV"),
|
||||
("imageonly_resnet50_proper", "classic_test", "baseline"),
|
||||
("imageonly_refugelike_proper", "classic_test", "pretrained"),
|
||||
("imageonly_resnet50_gtcrop_1.1", "classic_test", "GT crop 1.1x"),
|
||||
("imageonly_resnet50_gtcrop_2.5", "classic_test", "GT crop 2.5x"),
|
||||
("imageonly_resnet50_unetcrop_1.1","classic_test", "UNet crop 1.1x"),
|
||||
("imageonly_resnet50_unetcrop_2.5","classic_test", "UNet crop 2.5x"),
|
||||
]
|
||||
PHASE2_BASELINE = "imageonly_resnet50_proper"
|
||||
PHASE2_GROUPS = {
|
||||
"Backbone": ["imageonly_resnet50_proper", "imageonly_refugelike_proper"],
|
||||
"GT crop": ["imageonly_resnet50_proper", "imageonly_resnet50_gtcrop_1.1", "imageonly_resnet50_gtcrop_2.5"],
|
||||
"UNet crop": ["imageonly_resnet50_proper", "imageonly_resnet50_unetcrop_1.1", "imageonly_resnet50_unetcrop_2.5"],
|
||||
"Data leakage": ["imageonly_resnet50_leaky", "imageonly_resnet50_proper"],
|
||||
}
|
||||
|
||||
|
||||
def analyze_phase2(out_dir: Path):
|
||||
print("\n=== Phase 2 ===")
|
||||
aucs = {}
|
||||
for run, key, _ in PHASE2_RUNS:
|
||||
a = load_rep_aucs(RESULTS_ROOT / "phase2" / run, key)
|
||||
aucs[run] = a
|
||||
print(f" {run:40s} AUC={np.mean(a):.3f}±{np.std(a):.3f} n={len(a)}")
|
||||
|
||||
# Leakage impact
|
||||
leaky = aucs.get("imageonly_resnet50_leaky", np.array([]))
|
||||
proper = aucs.get("imageonly_resnet50_proper", np.array([]))
|
||||
if len(leaky) and len(proper):
|
||||
min_n = min(len(leaky), len(proper))
|
||||
p = wilcoxon_p(leaky[:min_n], proper[:min_n])
|
||||
delta = np.mean(leaky) - np.mean(proper)
|
||||
print(f"\n Data leakage inflates AUC by {delta:+.3f} (Wilcoxon p={p:.4f})")
|
||||
|
||||
# Build display order: baseline first, then non-baseline sorted by mean AUC descending
|
||||
base_run = PHASE2_BASELINE
|
||||
base_label = next(lbl for r, _, lbl in PHASE2_RUNS if r == base_run)
|
||||
others = [(r, lbl) for r, _, lbl in PHASE2_RUNS if r != base_run]
|
||||
others.sort(key=lambda x: -np.mean(aucs[x[0]]) if len(aucs.get(x[0], [])) else float("inf"))
|
||||
ordered = [(base_run, base_label)] + others
|
||||
|
||||
run_names = [r for r, _ in ordered]
|
||||
labels = [lbl for _, lbl in ordered]
|
||||
base_idx = 0
|
||||
base_aucs = aucs[base_run]
|
||||
|
||||
fig, ax = plt.subplots(figsize=(14, 6))
|
||||
fig.suptitle("Phase 2 — ResNet50: Backbone & Preprocessing Comparison", fontsize=12, fontweight="bold")
|
||||
|
||||
data = [aucs[r] for r in run_names]
|
||||
colors = ["#dd8452" if r == base_run else "#4c72b0" for r in run_names]
|
||||
x = np.arange(len(run_names))
|
||||
|
||||
bp = ax.boxplot(data, vert=True, patch_artist=True, positions=x,
|
||||
widths=0.3, showfliers=True,
|
||||
flierprops=dict(marker="o", markersize=3, alpha=0.5),
|
||||
medianprops=dict(color="#c44e52", linewidth=2))
|
||||
for patch, color in zip(bp["boxes"], colors):
|
||||
patch.set_facecolor(color)
|
||||
patch.set_alpha(0.8)
|
||||
|
||||
ax.set_xlim(-0.5, len(run_names) - 0.5)
|
||||
ax.set_ylabel("Test AUC", fontsize=11)
|
||||
ax.axhline(np.median(base_aucs), color="#dd8452", linewidth=1,
|
||||
linestyle="--", alpha=0.5, label="Baseline median")
|
||||
ax.legend(fontsize=9)
|
||||
ax.grid(axis="y", alpha=0.3)
|
||||
|
||||
# Build x-tick labels with p-value on a second line underneath
|
||||
tick_labels = []
|
||||
for i, run in enumerate(run_names):
|
||||
if run == base_run:
|
||||
tick_labels.append(labels[i])
|
||||
continue
|
||||
a = aucs[run]
|
||||
min_n = min(len(a), len(base_aucs))
|
||||
p = wilcoxon_p(a[:min_n], base_aucs[:min_n])
|
||||
p_str = f"p={p:.3f}" if not np.isnan(p) else "p=n/a"
|
||||
tick_labels.append(f"{labels[i]}\n{p_str}")
|
||||
|
||||
ax.set_xticks(x)
|
||||
ax.set_xticklabels(tick_labels, fontsize=10)
|
||||
|
||||
fig.tight_layout()
|
||||
path = out_dir / "phase2_analysis.png"
|
||||
fig.savefig(path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {path}")
|
||||
|
||||
|
||||
# ── Phase 3 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
PHASE3_GROUPS = {
|
||||
"Loss function": {
|
||||
"baseline": "Baseline (BCE fused)",
|
||||
"loss_all": "All losses",
|
||||
"loss_bcd_p03":"BCD p=0.3",
|
||||
"loss_bcd_p07":"BCD p=0.7",
|
||||
},
|
||||
"SE attention": {
|
||||
"baseline": "Baseline",
|
||||
"se_img_tower": "SE img tower",
|
||||
"se_cd_tower": "SE cd tower",
|
||||
"se_bridge": "SE bridge",
|
||||
"se_all": "SE all",
|
||||
},
|
||||
"IOP correction": {
|
||||
"baseline": "Baseline (none)",
|
||||
"iop_ratio": "Ratio",
|
||||
"iop_ratio_drop_raw":"Ratio + drop raw",
|
||||
"iop_ols": "OLS",
|
||||
"iop_lad": "LAD",
|
||||
"iop_multi": "Multi",
|
||||
},
|
||||
"Feature ablation": {
|
||||
"baseline": "Baseline (all)",
|
||||
"excl_iop": "Excl IOP",
|
||||
"excl_age": "Excl age",
|
||||
"excl_axial_length":"Excl axial length",
|
||||
"excl_refractive": "Excl refractive",
|
||||
},
|
||||
"Network dims": {
|
||||
"baseline": "Baseline",
|
||||
"cd_hidden_64": "CD hidden=64",
|
||||
"cd_hidden_256": "CD hidden=256",
|
||||
"fusion_dim_128": "Fusion dim=128",
|
||||
"fusion_dim_512": "Fusion dim=512",
|
||||
},
|
||||
"Dropout": {
|
||||
"baseline": "Baseline (0.5)",
|
||||
"bridge_dropout_03":"Bridge drop=0.3",
|
||||
"bridge_dropout_07":"Bridge drop=0.7",
|
||||
"cd_dropout_03": "CD drop=0.3",
|
||||
},
|
||||
"Backbone freezing": {
|
||||
"baseline": "Baseline (75%)",
|
||||
"freeze_25": "Freeze 25%",
|
||||
"freeze_50": "Freeze 50%",
|
||||
},
|
||||
"Warmup": {
|
||||
"baseline": "Baseline (cd40+twr3+fus3)",
|
||||
"warmup_no_cd": "No CD warmup",
|
||||
"warmup_tower5_fused5": "Tower5+Fused5",
|
||||
},
|
||||
"Sampling": {
|
||||
"baseline": "Baseline",
|
||||
"balanced_sampling": "Balanced sampling",
|
||||
},
|
||||
"Epoch length": {
|
||||
"epochs_1": "1 epoch",
|
||||
"epochs_5": "5 epochs",
|
||||
"epochs_10": "10 epochs",
|
||||
"epochs_20": "20 epochs",
|
||||
"epochs_30": "30 epochs (baseline)",
|
||||
"epochs_50": "50 epochs",
|
||||
},
|
||||
"Learning rate": {
|
||||
"baseline": "Baseline (1e-4)",
|
||||
"lr_3e4": "3e-4",
|
||||
"lr_1e3": "1e-3",
|
||||
"lr_1e5": "1e-5",
|
||||
},
|
||||
}
|
||||
PHASE3_BASELINE = "baseline"
|
||||
|
||||
|
||||
def analyze_phase3(out_dir: Path):
|
||||
print("\n=== Phase 3 ===")
|
||||
all_runs = set()
|
||||
for group in PHASE3_GROUPS.values():
|
||||
all_runs.update(group.keys())
|
||||
aucs = {}
|
||||
for run in all_runs:
|
||||
a = load_rep_aucs(RESULTS_ROOT / "phase3" / run, "classic_test")
|
||||
aucs[run] = a
|
||||
base_aucs = aucs[PHASE3_BASELINE]
|
||||
print(f" Baseline AUC: {np.mean(base_aucs):.3f}±{np.std(base_aucs):.3f}")
|
||||
|
||||
# Build display order: baseline first, then each group (non-baseline, sorted desc)
|
||||
GAP = 1.2 # extra space between groups
|
||||
pos = 0.0
|
||||
positions, box_data, tick_labels, colors, is_sig = [], [], [], [], []
|
||||
group_spans = [] # (x_mid, group_name) for title annotations
|
||||
|
||||
# Baseline box
|
||||
positions.append(pos)
|
||||
box_data.append(base_aucs)
|
||||
tick_labels.append("baseline")
|
||||
colors.append(C_BASELINE)
|
||||
is_sig.append(False)
|
||||
pos += 1 + GAP
|
||||
|
||||
def _group_max_median(group_runs):
|
||||
vals = [np.median(aucs[r]) for r in group_runs if r != PHASE3_BASELINE and len(aucs.get(r, []))]
|
||||
return max(vals) if vals else 0.0
|
||||
|
||||
sorted_groups = sorted(PHASE3_GROUPS.items(), key=lambda x: -_group_max_median(x[1]))
|
||||
|
||||
for shade_idx, (group_name, group_runs) in enumerate(sorted_groups):
|
||||
non_base = [(r, lbl) for r, lbl in group_runs.items() if r != PHASE3_BASELINE]
|
||||
non_base.sort(key=lambda x: -np.mean(aucs[x[0]]) if len(aucs.get(x[0], [])) else float("inf"))
|
||||
|
||||
group_start = pos
|
||||
for run, lbl in non_base:
|
||||
a = aucs.get(run, np.array([]))
|
||||
positions.append(pos)
|
||||
box_data.append(a)
|
||||
# p-value label under name
|
||||
min_n = min(len(a), len(base_aucs))
|
||||
p = wilcoxon_p(a[:min_n], base_aucs[:min_n]) if min_n >= 5 else float("nan")
|
||||
sig = not np.isnan(p) and p < 0.05
|
||||
if sig:
|
||||
tick_labels.append(f"* {lbl}\np={p:.3f}")
|
||||
else:
|
||||
tick_labels.append(lbl)
|
||||
colors.append(C_OTHER)
|
||||
is_sig.append(sig)
|
||||
pos += 1
|
||||
group_spans.append(((group_start + pos - 1) / 2, group_name, group_start, pos - 1, shade_idx))
|
||||
pos += GAP
|
||||
|
||||
fig, ax = plt.subplots(figsize=(9, 22))
|
||||
fig.suptitle("Phase 3 — Clinical Fusion Ablations (Single-Eye)",
|
||||
fontsize=FSIZE + 3, fontweight="bold")
|
||||
fig.subplots_adjust(top=0.97, left=0.38)
|
||||
|
||||
bp = ax.boxplot(box_data, vert=False, patch_artist=True, positions=positions,
|
||||
widths=0.5, showfliers=True,
|
||||
flierprops=dict(marker="o", markersize=3, alpha=0.5),
|
||||
medianprops=dict(color=C_MEDIAN, linewidth=2),
|
||||
manage_ticks=False)
|
||||
for patch, color in zip(bp["boxes"], colors):
|
||||
patch.set_facecolor(color)
|
||||
patch.set_alpha(0.8)
|
||||
|
||||
# Alternating shaded group backgrounds
|
||||
for x_mid, gname, g_start, g_end, shade_idx in group_spans:
|
||||
if shade_idx % 2 == 0:
|
||||
ax.axhspan(g_start - 0.5, g_end + 0.5, color="gray", alpha=0.07, zorder=0)
|
||||
# Group title to the left of the y-tick labels
|
||||
ax.text(-0.42, x_mid, gname, transform=ax.get_yaxis_transform(),
|
||||
ha="right", va="center", fontsize=FSIZE - 1, fontweight="bold", color="#444444")
|
||||
|
||||
ax.axvline(np.median(base_aucs), color=C_BASELINE, linewidth=1,
|
||||
linestyle="--", alpha=0.5, label="Baseline median")
|
||||
ax.set_yticks(positions)
|
||||
ax.set_yticklabels(tick_labels, fontsize=FSIZE - 1)
|
||||
for tick_lbl, sig in zip(ax.get_yticklabels(), is_sig):
|
||||
if sig:
|
||||
tick_lbl.set_fontweight("bold")
|
||||
ax.set_xlabel("Test AUC", fontsize=FSIZE + 1)
|
||||
ax.set_xlim(0.65, None)
|
||||
ax.set_ylim(-0.7, pos - GAP + 0.7)
|
||||
ax.invert_yaxis() # baseline at top
|
||||
ax.grid(axis="x", alpha=0.3)
|
||||
ax.legend(fontsize=FSIZE, loc="lower right")
|
||||
path = out_dir / "phase3_single_mode_ablations.png"
|
||||
fig.savefig(path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {path}")
|
||||
|
||||
# Print top winners vs baseline
|
||||
print("\n Top movers vs baseline (Wilcoxon, uncorrected):")
|
||||
deltas = []
|
||||
for run in aucs:
|
||||
if run == PHASE3_BASELINE:
|
||||
continue
|
||||
a = aucs[run]
|
||||
min_n = min(len(a), len(base_aucs))
|
||||
if min_n < 5:
|
||||
continue
|
||||
delta = np.mean(a) - np.mean(base_aucs)
|
||||
p = wilcoxon_p(a[:min_n], base_aucs[:min_n])
|
||||
deltas.append((run, delta, p))
|
||||
deltas.sort(key=lambda x: -x[1])
|
||||
for run, delta, p in deltas[:8]:
|
||||
print(f" {run:30s} {delta:+.3f} p={p:.4f} {stars(p)}")
|
||||
|
||||
# Pairwise table — IOP correction group (FDR-corrected Wilcoxon p-values)
|
||||
iop_runs = list(PHASE3_GROUPS["IOP correction"].keys())
|
||||
iop_labels = list(PHASE3_GROUPS["IOP correction"].values())
|
||||
_, corr = paired_matrix(iop_runs, aucs)
|
||||
reports_dir = out_dir / "reports"
|
||||
reports_dir.mkdir(exist_ok=True)
|
||||
import csv
|
||||
path2 = reports_dir / "phase3_iop_pairwise.csv"
|
||||
with open(path2, "w", newline="") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow([""] + iop_labels)
|
||||
for i, row_lbl in enumerate(iop_labels):
|
||||
cells = [row_lbl]
|
||||
for j in range(len(iop_labels)):
|
||||
p = corr[i, j]
|
||||
cells.append(f"{p:.4f} {stars(p)}" if not np.isnan(p) else "—")
|
||||
w.writerow(cells)
|
||||
print(f" Saved: {path2}")
|
||||
|
||||
|
||||
# ── Phase 4 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
PHASE4_RUNS = {
|
||||
"single": ("classic_test", "Single-eye\n(baseline)"),
|
||||
"ensemble": ("ensemble_test", "Ensemble\n(indep OD+OS)"),
|
||||
"bilateral": ("bilat_test", "BilateralHT\n(shared+concat)"),
|
||||
"siamese": ("bilat_test", "SiameseHT\n(mean+delta)"),
|
||||
"bilateral_loss_all": ("bilat_test", "BilateralHT\nall-losses"),
|
||||
"siamese_loss_all": ("bilat_test", "SiameseHT\nall-losses"),
|
||||
}
|
||||
PHASE4_BASELINE = "single"
|
||||
|
||||
|
||||
def analyze_phase4(out_dir: Path):
|
||||
print("\n=== Phase 4 ===")
|
||||
aucs = {}
|
||||
for run, (key, _) in PHASE4_RUNS.items():
|
||||
a = load_rep_aucs(RESULTS_ROOT / "phase4" / run, key)
|
||||
aucs[run] = a
|
||||
print(f" {run:25s} AUC={np.mean(a):.3f}±{np.std(a):.3f} n={len(a)}")
|
||||
|
||||
base_run = PHASE4_BASELINE
|
||||
base_aucs = aucs[base_run]
|
||||
others = [(r, PHASE4_RUNS[r][1]) for r in PHASE4_RUNS if r != base_run]
|
||||
others.sort(key=lambda x: -np.mean(aucs[x[0]]) if len(aucs.get(x[0], [])) else float("inf"))
|
||||
ordered = [(base_run, PHASE4_RUNS[base_run][1])] + others
|
||||
run_keys = [r for r, _ in ordered]
|
||||
labels = [lbl for _, lbl in ordered]
|
||||
data_list = [aucs[r] for r in run_keys]
|
||||
aucs_by_label = {lbl: aucs[r] for r, lbl in ordered}
|
||||
|
||||
fig, ax = plt.subplots(figsize=(11, 5))
|
||||
fig.suptitle("Phase 4 — Bilateral Architecture Comparison (Image Only)",
|
||||
fontsize=FSIZE + 2, fontweight="bold")
|
||||
boxplot_panel(ax, data_list, labels, base_idx=0,
|
||||
base_aucs=base_aucs, all_aucs_by_label=aucs_by_label)
|
||||
ax.set_ylim(0.75, None)
|
||||
|
||||
fig.tight_layout()
|
||||
path = out_dir / "phase4_analysis.png"
|
||||
fig.savefig(path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {path}")
|
||||
|
||||
|
||||
# ── Phase 5 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
PHASE5_RUNS = {
|
||||
# "single_fused": ("classic_test", "Single-eye (baseline)"),
|
||||
"ensemble_fused": ("ensemble_test", "Ensemble (baseline)"),
|
||||
"bilateral_fused": ("bilat_test", "BilateralHT"),
|
||||
"siamese_fused": ("bilat_test", "SiameseHT"),
|
||||
# "ensemble_fused_head": ("ensemble_test", "Ensemble\n+clinical+head"),
|
||||
"logit_mlp_head": ("ensemble_test", "Ensemble\n+Fusion head"),
|
||||
}
|
||||
PHASE5_BASELINE = "ensemble_fused"
|
||||
|
||||
|
||||
def analyze_phase5(out_dir: Path):
|
||||
print("\n=== Phase 5 ===")
|
||||
aucs = {}
|
||||
for run, (key, _) in PHASE5_RUNS.items():
|
||||
a = load_rep_aucs(RESULTS_ROOT / "phase5" / run, key)
|
||||
aucs[run] = a
|
||||
print(f" {run:25s} AUC={np.mean(a):.3f}±{np.std(a):.3f} n={len(a)}")
|
||||
|
||||
base_run = PHASE5_BASELINE
|
||||
base_aucs = aucs[base_run]
|
||||
others = [(r, PHASE5_RUNS[r][1]) for r in PHASE5_RUNS if r != base_run]
|
||||
others.sort(key=lambda x: -np.mean(aucs[x[0]]) if len(aucs.get(x[0], [])) else float("inf"))
|
||||
ordered = [(base_run, PHASE5_RUNS[base_run][1])] + others
|
||||
run_keys = [r for r, _ in ordered]
|
||||
labels = [lbl for _, lbl in ordered]
|
||||
data_list = [aucs[r] for r in run_keys]
|
||||
aucs_by_label = {lbl: aucs[r] for r, lbl in ordered}
|
||||
|
||||
fig, ax = plt.subplots(figsize=(11, 5))
|
||||
fig.suptitle("Phase 5 — Full HyperTower: Bilateral + Clinical",
|
||||
fontsize=FSIZE + 2, fontweight="bold")
|
||||
boxplot_panel(ax, data_list, labels, base_idx=0,
|
||||
base_aucs=base_aucs, all_aucs_by_label=aucs_by_label)
|
||||
ax.set_ylim(0.78, None)
|
||||
|
||||
fig.tight_layout()
|
||||
path = out_dir / "phase5_analysis.png"
|
||||
fig.savefig(path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {path}")
|
||||
|
||||
|
||||
# ── Cross-phase summary ───────────────────────────────────────────────────────
|
||||
|
||||
def analyze_cross_phase(out_dir: Path):
|
||||
"""Single figure tracing the best model from each phase."""
|
||||
print("\n=== Cross-phase progression ===")
|
||||
trajectory = [
|
||||
("Phase 2\nresnet50 proper", "phase2", "imageonly_resnet50_proper", "classic_test"),
|
||||
("Phase 2\nrefugelike proper", "phase2", "imageonly_refugelike_proper", "classic_test"),
|
||||
("Phase 3\n+IOP ratio\n+drop raw", "phase3", "iop_ratio_drop_raw", "classic_test"),
|
||||
("Phase 4\nensemble\n(image only)", "phase4", "ensemble", "ensemble_test"),
|
||||
("Phase 5\nensemble\n+clinical", "phase5", "ensemble_fused", "ensemble_test"),
|
||||
("Phase 5\nensemble\n+clinical+head","phase5","ensemble_fused_head", "ensemble_test"),
|
||||
]
|
||||
|
||||
labels, means, stds, all_aucs = [], [], [], []
|
||||
for lbl, phase, run, key in trajectory:
|
||||
a = load_rep_aucs(RESULTS_ROOT / phase / run, key)
|
||||
labels.append(lbl)
|
||||
means.append(np.mean(a) if len(a) else np.nan)
|
||||
stds.append(np.std(a) if len(a) else np.nan)
|
||||
all_aucs.append(a)
|
||||
print(f" {lbl.replace(chr(10),' '):35s} AUC={means[-1]:.3f}±{stds[-1]:.3f} n={len(a)}")
|
||||
|
||||
fig, ax = plt.subplots(figsize=(10, 4))
|
||||
x = np.arange(len(labels))
|
||||
ax.errorbar(x, means, yerr=stds, fmt="o-", linewidth=2, markersize=7,
|
||||
capsize=4, color="#4c72b0")
|
||||
ax.set_xticks(x)
|
||||
ax.set_xticklabels(labels, fontsize=8)
|
||||
ax.set_ylabel("Test AUC (10-rep mean ± std)")
|
||||
ax.set_title("HyperTower — Model Progression Across Phases", fontsize=12, fontweight="bold")
|
||||
ax.set_ylim(0.75, 0.95)
|
||||
ax.axhline(means[0], color="gray", linewidth=1, linestyle=":", alpha=0.5, label="Phase 2 baseline")
|
||||
ax.legend(fontsize=8)
|
||||
ax.grid(axis="y", alpha=0.3)
|
||||
|
||||
fig.tight_layout()
|
||||
path = out_dir / "cross_phase_progression.png"
|
||||
fig.savefig(path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {path}")
|
||||
|
||||
|
||||
# ── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--phase", type=int, choices=[1, 2, 3, 4, 5],
|
||||
help="Run only this phase (default: all)")
|
||||
ap.add_argument("--out", type=Path,
|
||||
default=Path(__file__).resolve().parents[3] / "v3" / "figures",
|
||||
help="Output directory for figures")
|
||||
args = ap.parse_args()
|
||||
|
||||
args.out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
run_all = args.phase is None
|
||||
if run_all or args.phase == 1:
|
||||
analyze_phase1(args.out)
|
||||
if run_all or args.phase == 2:
|
||||
analyze_phase2(args.out)
|
||||
if run_all or args.phase == 3:
|
||||
analyze_phase3(args.out)
|
||||
if run_all or args.phase == 4:
|
||||
analyze_phase4(args.out)
|
||||
if run_all or args.phase == 5:
|
||||
analyze_phase5(args.out)
|
||||
if run_all:
|
||||
analyze_cross_phase(args.out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,264 @@
|
||||
"""
|
||||
Phase 5 comparison panel — ROC curves + fusion event summaries.
|
||||
|
||||
Layout:
|
||||
Top row (1 × 3) — ROC curves: Single HyperTower | Bilateral Ensemble | Fused Head
|
||||
Bottom rows (2 × 1) — Fusion event summary (full-width) for Ensemble then Fused Head
|
||||
(Single mode has fused-only bridge; no meaningful fusion events)
|
||||
|
||||
All data derived from predictions_test.csv — no checkpoints required.
|
||||
|
||||
Usage:
|
||||
python -m v3.scripts.output_analysis.explainability.comparison_panel_phase5
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.patches as mpatches
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.metrics import roc_auc_score, roc_curve
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
RESULTS_ROOT = REPO_ROOT / "v3" / "results"
|
||||
FIGURES_ROOT = REPO_ROOT / "v3" / "figures" / "explainability"
|
||||
CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
|
||||
|
||||
RUNS = [
|
||||
{"label": "Single HyperTower", "run": "phase5/single_fused",
|
||||
"tower_path": "binary/single", "is_single": False},
|
||||
{"label": "Bilateral Ensemble", "run": "phase5/ensemble_fused",
|
||||
"tower_path": "binary/ensemble", "is_single": False},
|
||||
{"label": "Fused Head", "run": "phase5/logit_mlp_head",
|
||||
"tower_path": "binary/ensemble", "is_single": False},
|
||||
]
|
||||
|
||||
# Event taxonomy (img=image tower, md=clinical tower)
|
||||
_EVENT_KEYS = [
|
||||
"full_correction", "img_assist", "md_assist",
|
||||
"full_error", "img_drag", "md_drag",
|
||||
"concordant_correct", "concordant_wrong",
|
||||
]
|
||||
_EVENT_COLORS = [
|
||||
"#2ca02c", "#98df8a", "#b5cf6b", # positive
|
||||
"#d62728", "#ff9896", "#ffbb78", # negative
|
||||
"#aec7e8", "#c5b0d5", # concordant
|
||||
]
|
||||
_POSITIVE_KEYS = _EVENT_KEYS[:3]
|
||||
_NEGATIVE_KEYS = _EVENT_KEYS[3:6]
|
||||
_DISAGREE_KEYS = _POSITIVE_KEYS + _NEGATIVE_KEYS # exclude concordant
|
||||
|
||||
|
||||
# ── Data loading ──────────────────────────────────────────────────────────────
|
||||
|
||||
def load_pooled(run: str, tower_path: str) -> pd.DataFrame:
|
||||
run_dir = RESULTS_ROOT / run
|
||||
rows = []
|
||||
for rep in sorted(run_dir.glob("rep*")):
|
||||
tm = rep / tower_path
|
||||
if not tm.exists():
|
||||
continue
|
||||
for fold in sorted(tm.glob("fold[0-9]")):
|
||||
csv = fold / "predictions_test.csv"
|
||||
if csv.exists():
|
||||
df = pd.read_csv(csv)
|
||||
df["rep"] = rep.name
|
||||
df["fold"] = fold.name
|
||||
rows.append(df)
|
||||
if not rows:
|
||||
raise FileNotFoundError(f"No predictions found under {run_dir}/{tower_path}")
|
||||
return pd.concat(rows, ignore_index=True)
|
||||
|
||||
|
||||
def classify_events(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Add event_type column based on pred_fused/pred_img/pred_md vs y_true."""
|
||||
df = df.copy()
|
||||
y = df["y_true"].values
|
||||
pf = df["pred_fused"].values
|
||||
pi = df["pred_img"].values
|
||||
pm = df["pred_md"].values
|
||||
|
||||
fused_ok = pf == y
|
||||
img_ok = pi == y
|
||||
md_ok = pm == y
|
||||
|
||||
def _classify(fo, io, mo):
|
||||
if fo and io and mo: return "concordant_correct"
|
||||
if not fo and not io and not mo: return "concordant_wrong"
|
||||
if fo and not io and not mo: return "full_correction"
|
||||
if fo and io and not mo: return "img_assist"
|
||||
if fo and not io and mo: return "md_assist"
|
||||
if not fo and io and mo: return "full_error"
|
||||
if not fo and not io and mo: return "img_drag"
|
||||
if not fo and io and not mo: return "md_drag"
|
||||
return "other"
|
||||
|
||||
df["event_type"] = [_classify(fo, io, mo)
|
||||
for fo, io, mo in zip(fused_ok, img_ok, md_ok)]
|
||||
# conf_delta: fused prob minus average of img/md
|
||||
df["conf_fused"] = df["prob_fused_c1"]
|
||||
df["conf_img"] = df["prob_img_c1"]
|
||||
df["conf_md"] = df["prob_md_c1"]
|
||||
df["conf_delta"] = df["conf_fused"] - 0.5 * (df["conf_img"] + df["conf_md"])
|
||||
return df
|
||||
|
||||
|
||||
# ── ROC panel ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def _draw_roc(ax, df: pd.DataFrame, label: str, color: str) -> None:
|
||||
"""Draw per-fold ROC curves (faint) + mean ROC (bold) on ax."""
|
||||
fold_aucs = []
|
||||
for (rep, fold), grp in df.groupby(["rep", "fold"]):
|
||||
if grp["y_true"].nunique() < 2:
|
||||
continue
|
||||
fpr, tpr, _ = roc_curve(grp["y_true"], grp["prob_fused_c1"])
|
||||
ax.plot(fpr, tpr, color=color, alpha=0.12, lw=0.8)
|
||||
fold_aucs.append(roc_auc_score(grp["y_true"], grp["prob_fused_c1"]))
|
||||
|
||||
# Mean ROC via interpolation
|
||||
mean_fpr = np.linspace(0, 1, 200)
|
||||
tprs = []
|
||||
for (rep, fold), grp in df.groupby(["rep", "fold"]):
|
||||
if grp["y_true"].nunique() < 2:
|
||||
continue
|
||||
fpr, tpr, _ = roc_curve(grp["y_true"], grp["prob_fused_c1"])
|
||||
tprs.append(np.interp(mean_fpr, fpr, tpr))
|
||||
mean_tpr = np.mean(tprs, axis=0)
|
||||
mean_auc = np.mean(fold_aucs)
|
||||
std_auc = np.std(fold_aucs)
|
||||
ax.plot(mean_fpr, mean_tpr, color=color, lw=2.2,
|
||||
label=f"Mean AUC = {mean_auc:.3f} ± {std_auc:.3f}")
|
||||
ax.fill_between(mean_fpr,
|
||||
np.percentile(tprs, 25, axis=0),
|
||||
np.percentile(tprs, 75, axis=0),
|
||||
color=color, alpha=0.12)
|
||||
ax.plot([0, 1], [0, 1], "k--", lw=0.7, alpha=0.5)
|
||||
ax.set_xlim(-0.02, 1.02); ax.set_ylim(-0.02, 1.02)
|
||||
ax.set_xlabel("False Positive Rate", fontsize=9)
|
||||
ax.set_ylabel("True Positive Rate", fontsize=9)
|
||||
ax.set_title(label, fontsize=10, fontweight="bold")
|
||||
ax.legend(fontsize=8, loc="lower right")
|
||||
ax.grid(alpha=0.25)
|
||||
|
||||
|
||||
# ── Fusion summary panel ──────────────────────────────────────────────────────
|
||||
|
||||
def _draw_fusion_summary(axes_row, df: pd.DataFrame, label: str) -> None:
|
||||
"""Draw 3-panel fusion summary (disagreement events only) on axes_row (list of 3 axes)."""
|
||||
event_color = dict(zip(_EVENT_KEYS, _EVENT_COLORS))
|
||||
event_labels = {
|
||||
"full_correction": "Full correction\n(both wrong → right)",
|
||||
"img_assist": "Img assist\n(img✓ md✗ → right)",
|
||||
"md_assist": "MD assist\n(md✓ img✗ → right)",
|
||||
"full_error": "Full error\n(both right → wrong)",
|
||||
"img_drag": "Img drag\n(img✗ md✓ → wrong)",
|
||||
"md_drag": "MD drag\n(md✗ img✓ → wrong)",
|
||||
}
|
||||
|
||||
# Only count disagreement events (exclude concordant)
|
||||
counts = {k: (df["event_type"] == k).sum() for k in _DISAGREE_KEYS}
|
||||
|
||||
# Panel 0: totals bar (positive vs negative)
|
||||
ax = axes_row[0]
|
||||
for bar_x, keys in ((0, _POSITIVE_KEYS), (1, _NEGATIVE_KEYS)):
|
||||
bot = 0
|
||||
for k in keys:
|
||||
c = int(counts[k])
|
||||
ax.bar(bar_x, c, bottom=bot, color=event_color[k], width=0.5)
|
||||
if c > 0:
|
||||
ax.text(bar_x, bot + c / 2, str(c), ha="center", va="center",
|
||||
fontsize=8, fontweight="bold")
|
||||
bot += c
|
||||
ax.set_xticks([0, 1]); ax.set_xticklabels(["Positive\nevents", "Negative\nevents"])
|
||||
ax.set_ylabel("Count (all folds)")
|
||||
patches = [mpatches.Patch(color=event_color[k], label=event_labels[k].split("\n")[0])
|
||||
for k in _DISAGREE_KEYS if counts[k] > 0]
|
||||
ax.legend(handles=patches, fontsize=6, loc="upper right")
|
||||
ax.set_title(f"{label}\nDisagreement event totals", fontsize=9)
|
||||
|
||||
# Panel 1: per-fold stacked bar (disagreement events only)
|
||||
ax = axes_row[1]
|
||||
fold_groups = sorted(df.groupby(["rep", "fold"]), key=lambda x: x[0])
|
||||
x = np.arange(len(fold_groups))
|
||||
pos_bot = np.zeros(len(fold_groups))
|
||||
neg_bot = np.zeros(len(fold_groups))
|
||||
for k, color in zip(_POSITIVE_KEYS, _EVENT_COLORS[:3]):
|
||||
vals = np.array([(g["event_type"] == k).sum() for _, g in fold_groups], dtype=float)
|
||||
ax.bar(x, vals, bottom=pos_bot, color=color, width=0.6)
|
||||
pos_bot += vals
|
||||
for k, color in zip(_NEGATIVE_KEYS, _EVENT_COLORS[3:6]):
|
||||
vals = np.array([(g["event_type"] == k).sum() for _, g in fold_groups], dtype=float)
|
||||
ax.bar(x + 0.65, vals, bottom=neg_bot, color=color, width=0.6)
|
||||
neg_bot += vals
|
||||
ax.set_xticks([])
|
||||
ax.set_xlabel("Fold", fontsize=8)
|
||||
ax.set_ylabel("Count"); ax.set_title("Per-fold breakdown\n(left=positive, right=negative)", fontsize=9)
|
||||
|
||||
# Panel 2: img vs md confidence scatter (disagreement events only)
|
||||
ax = axes_row[2]
|
||||
for k in _DISAGREE_KEYS:
|
||||
sub = df[df["event_type"] == k]
|
||||
if len(sub) == 0:
|
||||
continue
|
||||
ax.scatter(sub["conf_img"], sub["conf_md"], c=event_color[k],
|
||||
alpha=0.65, s=30, edgecolors="none",
|
||||
label=event_labels[k].split("\n")[0])
|
||||
ax.plot([0, 1], [0, 1], "k--", lw=0.5, alpha=0.4)
|
||||
ax.set_xlabel("P(Glaucoma) — Image head"); ax.set_ylabel("P(Glaucoma) — MD head")
|
||||
ax.legend(fontsize=5.5, loc="lower right")
|
||||
ax.set_title("Tower confidence space\n(disagreement events only)", fontsize=9)
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
ROC_COLORS = ["#4c72b0", "#dd8452", "#55a868"]
|
||||
|
||||
print("Loading predictions ...")
|
||||
datasets = []
|
||||
for cfg, color in zip(RUNS, ROC_COLORS):
|
||||
df = load_pooled(cfg["run"], cfg["tower_path"])
|
||||
df = classify_events(df)
|
||||
datasets.append((cfg, df, color))
|
||||
|
||||
fusion_runs = [(cfg, df, color) for cfg, df, color in datasets]
|
||||
|
||||
# ── Layout ────────────────────────────────────────────────────────────────
|
||||
# Row 0: 3 ROC axes
|
||||
# Rows 1,2: 2 fusion summary strips (5 axes each, spanning full width)
|
||||
n_fusion = len(fusion_runs)
|
||||
fig = plt.figure(figsize=(20, 6 + 4.5 * n_fusion))
|
||||
gs = fig.add_gridspec(
|
||||
1 + n_fusion, 1,
|
||||
height_ratios=[5] + [4.5] * n_fusion,
|
||||
hspace=0.35,
|
||||
)
|
||||
|
||||
# ROC row — subdivide into 3
|
||||
roc_gs = gs[0].subgridspec(1, 3, wspace=0.28)
|
||||
for i, (cfg, df, color) in enumerate(datasets):
|
||||
ax = fig.add_subplot(roc_gs[i])
|
||||
_draw_roc(ax, df, cfg["label"], color)
|
||||
|
||||
# Fusion rows (3 panels each)
|
||||
for fi, (cfg, df, color) in enumerate(fusion_runs):
|
||||
fus_gs = gs[1 + fi].subgridspec(1, 3, wspace=0.32)
|
||||
axes_row = [fig.add_subplot(fus_gs[j]) for j in range(3)]
|
||||
_draw_fusion_summary(axes_row, df, cfg["label"])
|
||||
|
||||
fig.suptitle("Phase 5 — Model Comparison: ROC Curves & Fusion Event Analysis",
|
||||
fontsize=13, fontweight="bold", y=1.01)
|
||||
|
||||
out = FIGURES_ROOT / "comparison_panel_phase5.png"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,615 @@
|
||||
"""
|
||||
Phase 5 explainability — confidence strips and head comparison.
|
||||
|
||||
Works from saved prediction CSVs. If patient_id column is present (requires
|
||||
a re-run after the v3_hypertower.py update), points are colored by VFI
|
||||
severity group. Otherwise falls back to a single color per true class.
|
||||
|
||||
VFI severity groups (VF_MD from clinical data):
|
||||
Early VF_MD > -6
|
||||
Moderate VF_MD -6 to -12
|
||||
Severe VF_MD < -12
|
||||
|
||||
Produces (all in figures/explainability/):
|
||||
confidence_strips.png — vertical strip: P(glaucoma) by true class, VFI colored
|
||||
head_comparison.png — fused vs img vs md distributions side by side
|
||||
|
||||
Usage:
|
||||
python -m v3.scripts.output_analysis.explainability.confidence_strips
|
||||
python -m v3.scripts.output_analysis.explainability.confidence_strips \
|
||||
--run phase5/logit_mlp_head --clinical-dir Papila/ClinicalData
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.patches as mpatches
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
RESULTS_ROOT = REPO_ROOT / "v3" / "results"
|
||||
FIGURES_ROOT = REPO_ROOT / "v3" / "figures"
|
||||
CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
|
||||
|
||||
FONT = "DejaVu Sans"
|
||||
|
||||
# Colour palette
|
||||
C_NORMAL = "#78909C" # blue-grey — healthy controls (no VFI staging)
|
||||
C_EARLY = "#29B6F6" # sky-blue — glaucoma, early VFI loss
|
||||
C_MODERATE = "#FFB300" # amber — glaucoma, moderate VFI loss
|
||||
C_SEVERE = "#E53935" # vivid red — glaucoma, severe VFI loss
|
||||
C_UNKNOWN = "#BDBDBD" # light grey — glaucoma, VFI not recorded
|
||||
|
||||
SEV_LABELS = {
|
||||
"normal": "Normal",
|
||||
"early": "Glaucoma — early (VF_MD > −6)",
|
||||
"moderate": "Glaucoma — moderate (−12 to −6)",
|
||||
"severe": "Glaucoma — severe (VF_MD < −12)",
|
||||
"unknown": "Glaucoma — VF_MD not recorded",
|
||||
}
|
||||
SEV_COLORS = {
|
||||
"normal": C_NORMAL,
|
||||
"early": C_EARLY,
|
||||
"moderate": C_MODERATE,
|
||||
"severe": C_SEVERE,
|
||||
"unknown": C_UNKNOWN,
|
||||
}
|
||||
|
||||
HEAD_COLORS = {"fused": "#d4a017", "img": "#4e8d3a", "md": "#4c72b0"}
|
||||
HEAD_LABELS = {
|
||||
"fused": "Fused head",
|
||||
"img": "Image-only head",
|
||||
"md": "Clinical-only head",
|
||||
}
|
||||
|
||||
|
||||
# ── VFI data ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def load_vfi(clinical_dir: Path) -> pd.DataFrame:
|
||||
"""Return DataFrame with columns [patient_id (int), vf_md (float), severity (str)].
|
||||
Only includes patients in the binary study (Diagnosis 0=Normal, 1=Glaucoma).
|
||||
"""
|
||||
od = pd.read_excel(clinical_dir / "patient_data_od.xlsx", header=1)
|
||||
os_ = pd.read_excel(clinical_dir / "patient_data_os.xlsx", header=1)
|
||||
|
||||
def _clean(df, eye):
|
||||
df = df.copy()
|
||||
if "Patient ID" not in df.columns and "ID" in df.columns:
|
||||
df.rename(columns={"ID": "Patient ID"}, inplace=True)
|
||||
df["Patient ID"] = (
|
||||
df["Patient ID"].astype(str).str.extract(r"(\d+)")[0].astype(int)
|
||||
)
|
||||
df["Diagnosis"] = pd.to_numeric(df["Diagnosis"], errors="coerce")
|
||||
df["VF_MD"] = pd.to_numeric(df["VF_MD"], errors="coerce")
|
||||
# PAPILA: 0=Normal, 1=Glaucoma, 2=Suspect — keep only binary patients
|
||||
df = df[df["Diagnosis"].isin([0, 1])].copy()
|
||||
df["eye"] = eye
|
||||
return df[["Patient ID", "Diagnosis", "VF_MD", "eye"]]
|
||||
|
||||
combined = pd.concat([_clean(od, "OD"), _clean(os_, "OS")], ignore_index=True)
|
||||
|
||||
# Per patient: modal diagnosis, worst (most negative) VF_MD across eyes
|
||||
diag = (
|
||||
combined.groupby("Patient ID")["Diagnosis"]
|
||||
.agg(lambda x: x.mode().iloc[0])
|
||||
.reset_index()
|
||||
)
|
||||
vf = combined.groupby("Patient ID")["VF_MD"].min().reset_index()
|
||||
worst = diag.merge(vf, on="Patient ID").rename(
|
||||
columns={"Patient ID": "patient_id", "VF_MD": "vf_md", "Diagnosis": "diagnosis"}
|
||||
)
|
||||
|
||||
def _severity(row):
|
||||
if int(row["diagnosis"]) == 0:
|
||||
return "normal" # healthy control — no VFI staging
|
||||
v = row["vf_md"]
|
||||
if pd.isna(v):
|
||||
return "unknown" # glaucoma, no VFI recorded
|
||||
if v > -6:
|
||||
return "early"
|
||||
if v > -12:
|
||||
return "moderate"
|
||||
return "severe"
|
||||
|
||||
worst["severity"] = worst.apply(_severity, axis=1)
|
||||
return worst
|
||||
|
||||
|
||||
# ── Prediction loading ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def load_all_predictions(
|
||||
run_dir: Path, tower_path: str = "binary/ensemble"
|
||||
) -> pd.DataFrame:
|
||||
"""Pool predictions_test.csv across all reps and folds."""
|
||||
rows = []
|
||||
for rep_dir in sorted(run_dir.glob("rep*")):
|
||||
tm_dir = rep_dir / tower_path
|
||||
if not tm_dir.exists():
|
||||
continue
|
||||
for fold_dir in sorted(tm_dir.glob("fold[0-9]")):
|
||||
csv_path = fold_dir / "predictions_test.csv"
|
||||
if not csv_path.exists():
|
||||
continue
|
||||
df = pd.read_csv(csv_path)
|
||||
df["rep"] = rep_dir.name
|
||||
df["fold"] = fold_dir.name
|
||||
rows.append(df)
|
||||
if not rows:
|
||||
raise FileNotFoundError(f"No predictions_test.csv found under {run_dir}")
|
||||
return pd.concat(rows, ignore_index=True)
|
||||
|
||||
|
||||
# ── Figure 1: Confidence strips (vertical) ───────────────────────────────────
|
||||
|
||||
|
||||
def make_confidence_strips(
|
||||
df: pd.DataFrame, vfi: pd.DataFrame | None, out_path: Path
|
||||
) -> None:
|
||||
"""
|
||||
Vertical strip plot: x = true class, y = P(glaucoma).
|
||||
Points colored by VFI severity if patient_id column available, else uniform.
|
||||
"""
|
||||
rng = np.random.default_rng(42)
|
||||
has_vfi = (
|
||||
vfi is not None
|
||||
and "patient_id" in df.columns
|
||||
and df["patient_id"].notna().any()
|
||||
)
|
||||
|
||||
if has_vfi:
|
||||
df = df.copy()
|
||||
df["patient_id"] = pd.to_numeric(df["patient_id"], errors="coerce").astype(
|
||||
"Int64"
|
||||
)
|
||||
vfi_merge = vfi.copy()
|
||||
vfi_merge["patient_id"] = vfi_merge["patient_id"].astype("Int64")
|
||||
df = df.merge(
|
||||
vfi_merge[["patient_id", "severity"]], on="patient_id", how="left"
|
||||
)
|
||||
df["severity"] = df["severity"].fillna("unknown")
|
||||
else:
|
||||
df["severity"] = "unknown"
|
||||
|
||||
fig, ax = plt.subplots(figsize=(6, 7))
|
||||
fig.patch.set_facecolor("#e8e8e8")
|
||||
ax.set_facecolor("#e8e8e8")
|
||||
|
||||
x_pos = {0: 0.0, 1: 1.0}
|
||||
jitter_scale = 0.18
|
||||
|
||||
# Draw in severity order so severe is on top
|
||||
sev_order = ["normal", "unknown", "early", "moderate", "severe"]
|
||||
sev_alpha = {
|
||||
"normal": 0.40,
|
||||
"early": 0.55,
|
||||
"moderate": 0.70,
|
||||
"severe": 0.85,
|
||||
"unknown": 0.35,
|
||||
}
|
||||
sev_size = {"normal": 6, "early": 8, "moderate": 10, "severe": 12, "unknown": 6}
|
||||
|
||||
for sev in sev_order:
|
||||
mask = df["severity"] == sev
|
||||
if not mask.any():
|
||||
continue
|
||||
sub = df[mask]
|
||||
jitter = rng.uniform(-jitter_scale, jitter_scale, len(sub))
|
||||
x = np.array([x_pos[int(v)] for v in sub["y_true"]]) + jitter
|
||||
ax.scatter(
|
||||
x,
|
||||
sub["prob_fused_c1"].values,
|
||||
c=SEV_COLORS[sev],
|
||||
s=sev_size[sev],
|
||||
alpha=sev_alpha[sev],
|
||||
linewidths=0,
|
||||
zorder=3,
|
||||
label=SEV_LABELS[sev],
|
||||
)
|
||||
|
||||
# Median lines per class
|
||||
for cls, xc in x_pos.items():
|
||||
med = np.median(df.loc[df["y_true"] == cls, "prob_fused_c1"])
|
||||
ax.plot(
|
||||
[xc - jitter_scale - 0.04, xc + jitter_scale + 0.04],
|
||||
[med, med],
|
||||
color="#222",
|
||||
lw=2.0,
|
||||
zorder=5,
|
||||
)
|
||||
|
||||
ax.axhline(0.5, color="#888", lw=1.2, ls="--", alpha=0.7, zorder=2)
|
||||
ax.set_xticks([0, 1])
|
||||
ax.set_xticklabels(["Normal", "Glaucoma"], fontsize=11)
|
||||
ax.set_ylabel("Predicted P(Glaucoma)", fontsize=11)
|
||||
ax.set_ylim(-0.04, 1.04)
|
||||
ax.set_xlim(-0.55, 1.55)
|
||||
ax.set_title(
|
||||
"Confidence Strips — Fused Head\n(Phase 5, all folds)",
|
||||
fontsize=12,
|
||||
fontweight="bold",
|
||||
)
|
||||
ax.grid(axis="y", alpha=0.3, zorder=1)
|
||||
|
||||
# Legend — only show groups that appear
|
||||
handles, labels = ax.get_legend_handles_labels()
|
||||
if handles:
|
||||
ax.legend(
|
||||
handles=handles,
|
||||
labels=labels,
|
||||
fontsize=8.5,
|
||||
loc="upper center",
|
||||
framealpha=0.75,
|
||||
ncol=2,
|
||||
)
|
||||
|
||||
if not has_vfi:
|
||||
ax.text(
|
||||
0.98,
|
||||
0.02,
|
||||
"Re-run with updated v3_hypertower.py\nto enable VFI severity coloring",
|
||||
transform=ax.transAxes,
|
||||
ha="right",
|
||||
va="bottom",
|
||||
fontsize=7.5,
|
||||
color="#888",
|
||||
style="italic",
|
||||
)
|
||||
|
||||
fig.tight_layout()
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out_path, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {out_path}")
|
||||
|
||||
|
||||
# ── Figure 2: Head comparison ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def make_head_comparison(df: pd.DataFrame, out_path: Path) -> None:
|
||||
"""Side-by-side violin + strip of P(glaucoma) by true class for each head."""
|
||||
heads = ["fused", "img", "md"]
|
||||
prob_cols = {"fused": "prob_fused_c1", "img": "prob_img_c1", "md": "prob_md_c1"}
|
||||
|
||||
rng = np.random.default_rng(42)
|
||||
fig, axes = plt.subplots(1, 3, figsize=(11, 4.5), sharey=True)
|
||||
fig.patch.set_facecolor("#e8e8e8")
|
||||
fig.suptitle(
|
||||
"Head Comparison — P(Glaucoma) by True Class (Phase 5, all folds)",
|
||||
fontsize=12,
|
||||
fontweight="bold",
|
||||
)
|
||||
|
||||
c_normal = "#4c72b0"
|
||||
c_glaucoma = "#c44e52"
|
||||
|
||||
for ax, head in zip(axes, heads):
|
||||
ax.set_facecolor("#e8e8e8")
|
||||
col = prob_cols[head]
|
||||
data_by_class = [df.loc[df["y_true"] == cls, col].values for cls in [0, 1]]
|
||||
|
||||
vp = ax.violinplot(
|
||||
data_by_class,
|
||||
positions=[0, 1],
|
||||
widths=0.6,
|
||||
showmedians=True,
|
||||
showextrema=False,
|
||||
)
|
||||
for body, color in zip(vp["bodies"], [c_normal, c_glaucoma]):
|
||||
body.set_facecolor(color)
|
||||
body.set_alpha(0.35)
|
||||
vp["cmedians"].set_color("#222")
|
||||
vp["cmedians"].set_linewidth(2)
|
||||
|
||||
for cls, color in zip([0, 1], [c_normal, c_glaucoma]):
|
||||
vals = data_by_class[cls]
|
||||
jitter = rng.uniform(-0.12, 0.12, len(vals))
|
||||
ax.scatter(
|
||||
cls + jitter, vals, color=color, s=4, alpha=0.30, linewidths=0, zorder=3
|
||||
)
|
||||
|
||||
ax.axhline(0.5, color="#888", lw=1.0, ls="--", alpha=0.6)
|
||||
ax.set_xticks([0, 1])
|
||||
ax.set_xticklabels(["Normal", "Glaucoma"], fontsize=9)
|
||||
ax.set_title(
|
||||
HEAD_LABELS[head], fontsize=10, fontweight="bold", color=HEAD_COLORS[head]
|
||||
)
|
||||
ax.set_ylim(-0.05, 1.05)
|
||||
ax.grid(axis="y", alpha=0.3)
|
||||
if head == "fused":
|
||||
ax.set_ylabel("Predicted P(Glaucoma)", fontsize=10)
|
||||
|
||||
from sklearn.metrics import roc_auc_score
|
||||
|
||||
try:
|
||||
auc = roc_auc_score(df["y_true"], df[col])
|
||||
ax.text(
|
||||
0.97,
|
||||
0.04,
|
||||
f"AUC = {auc:.3f}",
|
||||
transform=ax.transAxes,
|
||||
ha="right",
|
||||
va="bottom",
|
||||
fontsize=9,
|
||||
color="#333",
|
||||
bbox=dict(facecolor="white", alpha=0.65, edgecolor="none", pad=2),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
fig.tight_layout()
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out_path, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {out_path}")
|
||||
|
||||
|
||||
# ── Multi-model comparison strip ──────────────────────────────────────────────
|
||||
|
||||
|
||||
def make_comparison_strips(
|
||||
run_configs: list[dict], vfi: pd.DataFrame, out_path: Path
|
||||
) -> None:
|
||||
"""
|
||||
Side-by-side confidence strips for multiple runs.
|
||||
Each config: {"label": str, "df": DataFrame}.
|
||||
"""
|
||||
from sklearn.metrics import roc_auc_score
|
||||
|
||||
rng = np.random.default_rng(42)
|
||||
|
||||
n = len(run_configs)
|
||||
fig, axes = plt.subplots(1, n, figsize=(4.5 * n, 7), sharey=True)
|
||||
if n == 1:
|
||||
axes = [axes]
|
||||
fig.patch.set_facecolor("#e8e8e8")
|
||||
fig.suptitle(
|
||||
"Confidence Strips by Model (Phase 5, all folds)",
|
||||
fontsize=13,
|
||||
fontweight="bold",
|
||||
)
|
||||
|
||||
sev_order = ["normal", "unknown", "early", "moderate", "severe"]
|
||||
sev_alpha = {
|
||||
"normal": 0.40,
|
||||
"early": 0.55,
|
||||
"moderate": 0.70,
|
||||
"severe": 0.85,
|
||||
"unknown": 0.35,
|
||||
}
|
||||
sev_size = {"normal": 6, "early": 8, "moderate": 10, "severe": 12, "unknown": 6}
|
||||
x_pos = {0: 0.0, 1: 1.0}
|
||||
jitter_scale = 0.18
|
||||
|
||||
for ax, cfg in zip(axes, run_configs):
|
||||
ax.set_facecolor("#e8e8e8")
|
||||
df = cfg["df"]
|
||||
|
||||
# Attach VFI severity
|
||||
df = df.copy()
|
||||
df["patient_id"] = pd.to_numeric(df["patient_id"], errors="coerce").astype(
|
||||
"Int64"
|
||||
)
|
||||
vfi_m = vfi.copy()
|
||||
vfi_m["patient_id"] = vfi_m["patient_id"].astype("Int64")
|
||||
df = df.merge(vfi_m[["patient_id", "severity"]], on="patient_id", how="left")
|
||||
df["severity"] = df["severity"].fillna("unknown")
|
||||
|
||||
for sev in sev_order:
|
||||
mask = df["severity"] == sev
|
||||
if not mask.any():
|
||||
continue
|
||||
sub = df[mask]
|
||||
jitter = rng.uniform(-jitter_scale, jitter_scale, len(sub))
|
||||
x = np.array([x_pos[int(v)] for v in sub["y_true"]]) + jitter
|
||||
ax.scatter(
|
||||
x,
|
||||
sub["prob_fused_c1"].values,
|
||||
c=SEV_COLORS[sev],
|
||||
s=sev_size[sev],
|
||||
alpha=sev_alpha[sev],
|
||||
linewidths=0,
|
||||
zorder=3,
|
||||
)
|
||||
|
||||
# IQR box + median line per class
|
||||
iqr_w = 0.06
|
||||
xticklabels = []
|
||||
for cls, xc in x_pos.items():
|
||||
vals = df.loc[df["y_true"] == cls, "prob_fused_c1"]
|
||||
med = np.median(vals)
|
||||
q25 = np.percentile(vals, 25)
|
||||
q75 = np.percentile(vals, 75)
|
||||
# Subtle translucent IQR box
|
||||
ax.add_patch(
|
||||
plt.Rectangle(
|
||||
(xc - iqr_w, q25),
|
||||
2 * iqr_w,
|
||||
q75 - q25,
|
||||
facecolor="#555",
|
||||
alpha=0.18,
|
||||
linewidth=0,
|
||||
zorder=4,
|
||||
)
|
||||
)
|
||||
# Median line
|
||||
ax.plot(
|
||||
[xc - jitter_scale - 0.04, xc + jitter_scale + 0.04],
|
||||
[med, med],
|
||||
color="#222",
|
||||
lw=2.0,
|
||||
zorder=5,
|
||||
label="Median" if cls == 0 else None,
|
||||
)
|
||||
# TP / TN rate below x-label
|
||||
if cls == 1:
|
||||
rate = (vals > 0.5).mean() * 100
|
||||
xticklabels.append(f"Glaucoma\nTP {rate:.0f}%")
|
||||
else:
|
||||
rate = (vals <= 0.5).mean() * 100
|
||||
xticklabels.append(f"Normal\nTN {rate:.0f}%")
|
||||
|
||||
ax.axhline(0.5, color="#888", lw=1.2, ls="--", alpha=0.7, zorder=2)
|
||||
ax.set_xticks([0, 1])
|
||||
ax.set_xticklabels(xticklabels, fontsize=10)
|
||||
ax.set_title(cfg["label"], fontsize=11, fontweight="bold")
|
||||
ax.set_ylim(-0.04, 1.04)
|
||||
ax.set_xlim(-0.55, 1.55)
|
||||
ax.grid(axis="y", alpha=0.3, zorder=1)
|
||||
|
||||
try:
|
||||
fold_aucs = [
|
||||
roc_auc_score(g["y_true"], g["prob_fused_c1"])
|
||||
for _, g in df.groupby(["rep", "fold"])
|
||||
if g["y_true"].nunique() > 1
|
||||
]
|
||||
mean_auc = np.mean(fold_aucs)
|
||||
std_auc = np.std(fold_aucs)
|
||||
ax.text(
|
||||
0.65,
|
||||
0.0,
|
||||
f"AUC = {mean_auc:.3f} ± {std_auc:.3f}",
|
||||
transform=ax.transAxes,
|
||||
ha="right",
|
||||
va="bottom",
|
||||
fontsize=9,
|
||||
color="#333",
|
||||
bbox=dict(facecolor="white", alpha=0.65, edgecolor="none", pad=2),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
axes[0].set_ylabel("Predicted P(Glaucoma)", fontsize=11)
|
||||
|
||||
# Shared legend
|
||||
legend_patches = [
|
||||
mpatches.Patch(color=SEV_COLORS[s], label=SEV_LABELS[s])
|
||||
for s in ["normal", "early", "moderate", "severe", "unknown"]
|
||||
]
|
||||
fig.legend(
|
||||
handles=legend_patches,
|
||||
fontsize=9,
|
||||
loc="lower center",
|
||||
ncol=len(legend_patches),
|
||||
framealpha=0.75,
|
||||
bbox_to_anchor=(0.5, -0.02),
|
||||
)
|
||||
|
||||
fig.tight_layout(rect=[0, 0.06, 1, 1])
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out_path, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {out_path}")
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Default runs shown in the comparison
|
||||
DEFAULT_RUNS = [
|
||||
{
|
||||
"run": "phase4/single",
|
||||
"tower_path": "binary/single",
|
||||
"label": "Single HyperTower",
|
||||
},
|
||||
{
|
||||
"run": "phase5/ensemble_fused",
|
||||
"tower_path": "binary/ensemble",
|
||||
"label": "Bilateral Ensemble",
|
||||
},
|
||||
{
|
||||
"run": "phase5/logit_mlp_head",
|
||||
"tower_path": "binary/ensemble",
|
||||
"label": "Fused Head",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _aggregate_eye_to_patient(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Single-mode predictions are eye-level (2 rows per patient per fold).
|
||||
In the test loader, OD rows come first (sorted patient ID order) then OS.
|
||||
Average the two eyes to get one patient-level row per fold.
|
||||
"""
|
||||
rows = []
|
||||
prob_cols = [c for c in df.columns if c.startswith("prob_")]
|
||||
pred_cols = [c for c in df.columns if c.startswith("pred_")]
|
||||
|
||||
for (rep, fold), grp in df.groupby(["rep", "fold"]):
|
||||
n = len(grp)
|
||||
half = n // 2
|
||||
od = grp.iloc[:half].reset_index(drop=True)
|
||||
os_ = grp.iloc[half:].reset_index(drop=True)
|
||||
pat = od.copy()
|
||||
for col in prob_cols:
|
||||
pat[col] = (od[col].values + os_[col].values) / 2
|
||||
for col in pred_cols:
|
||||
pat[col] = (pat[col.replace("pred_", "prob_") + "_c1"] >= 0.5).astype(int)
|
||||
rows.append(pat)
|
||||
|
||||
return pd.concat(rows, ignore_index=True)
|
||||
|
||||
|
||||
def _load_run(run: str, tower_path: str, clinical_dir: Path) -> pd.DataFrame:
|
||||
run_dir = RESULTS_ROOT / run
|
||||
print(f" Loading {run} ...")
|
||||
df = load_all_predictions(run_dir, tower_path=tower_path)
|
||||
if tower_path.endswith("/single"):
|
||||
from v3.scripts.output_analysis.explainability.fold_patient_ids import (
|
||||
attach_patient_ids_single,
|
||||
)
|
||||
|
||||
df = attach_patient_ids_single(df, clinical_dir=clinical_dir, batch_size=8)
|
||||
else:
|
||||
from v3.scripts.output_analysis.explainability.fold_patient_ids import (
|
||||
attach_patient_ids,
|
||||
)
|
||||
|
||||
df = attach_patient_ids(df, clinical_dir=clinical_dir)
|
||||
return df
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(
|
||||
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
||||
)
|
||||
ap.add_argument(
|
||||
"--run", default="phase5/logit_mlp_head", help="Single run for standalone plots"
|
||||
)
|
||||
ap.add_argument("--tower-path", default="binary/ensemble")
|
||||
ap.add_argument("--clinical-dir", type=Path, default=CLINICAL_DIR)
|
||||
ap.add_argument("--out", type=Path, default=FIGURES_ROOT / "explainability")
|
||||
args = ap.parse_args()
|
||||
|
||||
print("Loading VFI data ...")
|
||||
vfi = load_vfi(args.clinical_dir)
|
||||
|
||||
# ── Single-run plots (strips + head comparison) ──────────────────────────
|
||||
df = _load_run(args.run, args.tower_path, args.clinical_dir)
|
||||
make_confidence_strips(df, vfi, args.out / "confidence_strips.png")
|
||||
make_head_comparison(df, args.out / "head_comparison.png")
|
||||
|
||||
# ── Multi-model comparison ───────────────────────────────────────────────
|
||||
print("Building comparison strips ...")
|
||||
run_configs = []
|
||||
for cfg in DEFAULT_RUNS:
|
||||
try:
|
||||
df_r = _load_run(cfg["run"], cfg["tower_path"], args.clinical_dir)
|
||||
run_configs.append({"label": cfg["label"], "df": df_r})
|
||||
except FileNotFoundError as e:
|
||||
print(f" Skipping {cfg['run']}: {e}")
|
||||
if run_configs:
|
||||
make_comparison_strips(
|
||||
run_configs, vfi, args.out / "confidence_strips_comparison.png"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,181 @@
|
||||
"""
|
||||
Derives test-set patient IDs for any (rep, fold) without re-running training.
|
||||
|
||||
The split is fully deterministic: fold_seed = rep_seed_start + rep * rep_seed_step.
|
||||
build_samples() groups by patient_id with sort=True (pandas default), and the
|
||||
test DataLoader uses shuffle=False — so rows in predictions_test.csv are always
|
||||
in ascending Patient ID order within each test fold.
|
||||
|
||||
Usage:
|
||||
from v3.scripts.output_analysis.explainability.fold_patient_ids import get_test_patient_ids
|
||||
pids = get_test_patient_ids(rep=0, fold=2) # list of int patient IDs, sorted
|
||||
|
||||
# Attach to a pooled predictions DataFrame:
|
||||
df = attach_patient_ids(df, clinical_dir=...)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
|
||||
|
||||
# These match the defaults in run_cv.py
|
||||
_REP_SEED_START = 100
|
||||
_REP_SEED_STEP = 100
|
||||
_N_SPLITS = 5
|
||||
_EVAL_MODE = "binary"
|
||||
_LABEL_COL = "Diagnosis"
|
||||
_PATIENT_COL = "Patient ID"
|
||||
|
||||
|
||||
@lru_cache(maxsize=4)
|
||||
def _load_clinical(clinical_dir: Path) -> pd.DataFrame:
|
||||
"""
|
||||
Load OD/OS Excel sheets, extract Patient ID + Diagnosis, binary-filter.
|
||||
Returns a DataFrame with one row per eye (OD+OS stacked), columns:
|
||||
[Patient ID, Diagnosis, eyeID, VF_MD].
|
||||
"""
|
||||
od = pd.read_excel(clinical_dir / "patient_data_od.xlsx", header=1)
|
||||
os_ = pd.read_excel(clinical_dir / "patient_data_os.xlsx", header=1)
|
||||
od["eyeID"] = "OD"
|
||||
os_["eyeID"] = "OS"
|
||||
df = pd.concat([od, os_], ignore_index=True)
|
||||
# Raw column is "ID" (e.g. "#002"); canonicalize to "Patient ID"
|
||||
if "Patient ID" not in df.columns and "ID" in df.columns:
|
||||
df.rename(columns={"ID": "Patient ID"}, inplace=True)
|
||||
df["Patient ID"] = df["Patient ID"].astype(str).str.extract(r"(\d+)")[0].astype(int)
|
||||
df["Diagnosis"] = pd.to_numeric(df["Diagnosis"], errors="coerce")
|
||||
df["VF_MD"] = pd.to_numeric(df["VF_MD"], errors="coerce")
|
||||
# PAPILA encoding: 0=Normal, 1=Glaucoma, 2=Suspect
|
||||
# Binary mode keeps 0 and 1, excludes Suspect (2)
|
||||
df = df[df["Diagnosis"].isin([0, 1])].copy()
|
||||
return df.reset_index(drop=True)
|
||||
|
||||
|
||||
def _build_splits(clinical_dir: Path, fold_seed: int) -> list[Any]:
|
||||
"""Return list of PatientSplit for a given fold seed."""
|
||||
import sys
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
from v3.classes.split_manager import PatientFirstSplitManager, build_patient_split_plans
|
||||
|
||||
df = _load_clinical(clinical_dir)
|
||||
|
||||
# Patient-level label table (mode label per patient)
|
||||
patient_table = (
|
||||
df.groupby(_PATIENT_COL)[_LABEL_COL]
|
||||
.agg(lambda x: x.mode().iloc[0])
|
||||
.reset_index()
|
||||
)
|
||||
plans_raw = build_patient_split_plans(
|
||||
patient_ids=patient_table[_PATIENT_COL].to_numpy(),
|
||||
patient_labels=patient_table[_LABEL_COL].to_numpy(),
|
||||
n_splits=_N_SPLITS,
|
||||
seed=fold_seed,
|
||||
)
|
||||
|
||||
# Wrap into PatientSplit-like objects with .test DataFrame
|
||||
class _Split:
|
||||
def __init__(self, test_ids):
|
||||
self.test = df[df[_PATIENT_COL].isin(test_ids)].reset_index(drop=True)
|
||||
|
||||
return [_Split(p.test_patient_ids) for p in plans_raw]
|
||||
|
||||
|
||||
def get_test_patient_ids(rep: int, fold: int,
|
||||
clinical_dir: Path = CLINICAL_DIR,
|
||||
rep_seed_start: int = _REP_SEED_START,
|
||||
rep_seed_step: int = _REP_SEED_STEP) -> list[int]:
|
||||
"""
|
||||
Return sorted list of Patient IDs in the test set for (rep, fold).
|
||||
Matches the row order of predictions_test.csv for that fold.
|
||||
"""
|
||||
fold_seed = rep_seed_start + rep * rep_seed_step
|
||||
plans = _build_splits(clinical_dir, fold_seed)
|
||||
test_df = plans[fold].test
|
||||
# groupby sorts by default → same order as build_samples / test loader
|
||||
return sorted(test_df[_PATIENT_COL].unique().tolist())
|
||||
|
||||
|
||||
def _row_to_patient_pos(row_idx: int, n_patients: int, batch_size: int) -> int:
|
||||
"""
|
||||
Map a single-mode row index to its patient position in the sorted patient list.
|
||||
|
||||
collect_probs_single_components (aggregate_patient=False) emits predictions
|
||||
in batch-interleaved order: for each batch of B patients, OD rows come first
|
||||
then OS rows. The last batch may be smaller than batch_size.
|
||||
|
||||
Batch i (B patients): rows [i*2B .. i*2B+B-1] = OD
|
||||
[i*2B+B .. i*2B+2B-1] = OS
|
||||
Patient position = i*B + (row_in_batch % B)
|
||||
"""
|
||||
full = n_patients // batch_size
|
||||
last_b = n_patients % batch_size
|
||||
for bi in range(full):
|
||||
s = bi * 2 * batch_size
|
||||
if s <= row_idx < s + 2 * batch_size:
|
||||
return bi * batch_size + (row_idx - s) % batch_size
|
||||
if last_b > 0:
|
||||
s = full * 2 * batch_size
|
||||
return full * batch_size + (row_idx - s) % last_b
|
||||
raise IndexError(f"row_idx {row_idx} out of range for n_patients={n_patients}")
|
||||
|
||||
|
||||
def attach_patient_ids(df: pd.DataFrame,
|
||||
clinical_dir: Path = CLINICAL_DIR,
|
||||
rep_seed_start: int = _REP_SEED_START,
|
||||
rep_seed_step: int = _REP_SEED_STEP) -> pd.DataFrame:
|
||||
"""
|
||||
Add a 'patient_id' column to a pooled predictions DataFrame.
|
||||
Requires 'rep' and 'fold' columns (added by load_all_predictions).
|
||||
The 'idx' column is the row index within each fold's test set.
|
||||
For patient-level modes (ensemble): idx == patient position directly.
|
||||
"""
|
||||
df = df.copy()
|
||||
pid_col = []
|
||||
|
||||
for _, row in df.iterrows():
|
||||
rep_idx = int(row["rep"].replace("rep", ""))
|
||||
fold_idx = int(row["fold"].replace("fold", ""))
|
||||
idx = int(row["idx"])
|
||||
pids = get_test_patient_ids(rep_idx, fold_idx,
|
||||
clinical_dir=clinical_dir,
|
||||
rep_seed_start=rep_seed_start,
|
||||
rep_seed_step=rep_seed_step)
|
||||
pid_col.append(pids[idx] if idx < len(pids) else None)
|
||||
|
||||
df["patient_id"] = pid_col
|
||||
return df
|
||||
|
||||
|
||||
def attach_patient_ids_single(df: pd.DataFrame,
|
||||
clinical_dir: Path = CLINICAL_DIR,
|
||||
batch_size: int = 8,
|
||||
rep_seed_start: int = _REP_SEED_START,
|
||||
rep_seed_step: int = _REP_SEED_STEP) -> pd.DataFrame:
|
||||
"""
|
||||
Like attach_patient_ids but for single (eye-level) mode.
|
||||
Single mode emits predictions in batch-interleaved order (see _row_to_patient_pos).
|
||||
batch_size must match the --batch-size used during training (default 8).
|
||||
"""
|
||||
df = df.copy()
|
||||
pid_col = []
|
||||
|
||||
for _, row in df.iterrows():
|
||||
rep_idx = int(row["rep"].replace("rep", ""))
|
||||
fold_idx = int(row["fold"].replace("fold", ""))
|
||||
idx = int(row["idx"])
|
||||
pids = get_test_patient_ids(rep_idx, fold_idx,
|
||||
clinical_dir=clinical_dir,
|
||||
rep_seed_start=rep_seed_start,
|
||||
rep_seed_step=rep_seed_step)
|
||||
patient_pos = _row_to_patient_pos(idx, len(pids), batch_size)
|
||||
pid_col.append(pids[patient_pos])
|
||||
|
||||
df["patient_id"] = pid_col
|
||||
return df
|
||||
@@ -0,0 +1,569 @@
|
||||
"""
|
||||
GradCAM analysis for Phase 5 — logit_mlp_head checkpointed run.
|
||||
|
||||
Produces (all in figures/explainability/gradcam/):
|
||||
mean_cam_normal.png — average heatmap across all normal test eyes
|
||||
mean_cam_glaucoma.png — average heatmap across all glaucoma test eyes
|
||||
mean_cam_comparison.png — side-by-side normal vs glaucoma mean CAMs
|
||||
overlay_grid_normal.png — grid of individual overlays (normal eyes)
|
||||
overlay_grid_glaucoma.png — grid of individual overlays (glaucoma eyes)
|
||||
|
||||
Checkpoints loaded from:
|
||||
v3/results/phase5/logit_mlp_head_ckpt/rep00/binary/ensemble/fold{0..4}/best_single.pt
|
||||
|
||||
Usage:
|
||||
python -m v3.scripts.output_analysis.explainability.gradcam_phase5
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.cm as cm
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from PIL import Image
|
||||
from tqdm import tqdm
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
CKPT_RUN = REPO_ROOT / "v3" / "results" / "phase5" / "logit_mlp_head_ckpt"
|
||||
FIGURES_ROOT = REPO_ROOT / "v3" / "figures" / "explainability" / "gradcam"
|
||||
CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
|
||||
IMAGE_DIR = REPO_ROOT / "Papila" / "FundusImages"
|
||||
CONTOUR_DIR = REPO_ROOT / "Papila" / "ExpertsSegmentations" / "Contours"
|
||||
|
||||
DISC_SPAN = 5 # patch side = DISC_SPAN × disc diameter
|
||||
PATCH_SIZE = 96 # output thumbnail pixels
|
||||
|
||||
# Model hyperparameters (inferred from checkpoint weight shapes)
|
||||
BACKBONE = "resnet50"
|
||||
NUM_CLASSES = 2
|
||||
CD_HIDDEN = 128
|
||||
FUSION_DIM = 256
|
||||
|
||||
LABEL_NAMES = {0: "Normal", 1: "Glaucoma"}
|
||||
|
||||
|
||||
# ── GradCAM ──────────────────────────────────────────────────────────────────
|
||||
|
||||
class GradCAM:
|
||||
"""Minimal GradCAM via forward/backward hooks."""
|
||||
|
||||
def __init__(self, target_layer: torch.nn.Module) -> None:
|
||||
self._acts = None
|
||||
self._grads = None
|
||||
self._h1 = target_layer.register_forward_hook(self._save_acts)
|
||||
self._h2 = target_layer.register_full_backward_hook(self._save_grads)
|
||||
|
||||
def _save_acts(self, _m, _i, output):
|
||||
self._acts = output.detach()
|
||||
|
||||
def _save_grads(self, _m, _gi, grad_output):
|
||||
self._grads = grad_output[0].detach()
|
||||
|
||||
def compute(self, img: torch.Tensor, meta: torch.Tensor,
|
||||
model: torch.nn.Module, target_class: int | None = None) -> tuple[np.ndarray, int]:
|
||||
"""Return (cam [H,W] normalised 0-1, predicted_class)."""
|
||||
model.eval()
|
||||
with torch.enable_grad():
|
||||
out = model(img, meta)
|
||||
pred = int(out.argmax(1).item())
|
||||
tc = pred if target_class is None else target_class
|
||||
model.zero_grad()
|
||||
out[0, tc].backward()
|
||||
|
||||
weights = self._grads.mean(dim=(2, 3), keepdim=True)
|
||||
cam = F.relu((weights * self._acts).sum(dim=1, keepdim=True))
|
||||
cam = F.interpolate(cam, img.shape[-2:], mode="bilinear", align_corners=False)
|
||||
cam_np = cam.squeeze().cpu().numpy()
|
||||
lo, hi = cam_np.min(), cam_np.max()
|
||||
return (cam_np - lo) / (hi - lo + 1e-8), pred
|
||||
|
||||
def remove(self) -> None:
|
||||
self._h1.remove(); self._h2.remove()
|
||||
|
||||
|
||||
def overlay_gradcam(pil: Image.Image, cam: np.ndarray, alpha: float = 0.45) -> Image.Image:
|
||||
cam_u8 = (cam * 255).astype(np.uint8)
|
||||
cam_r = np.array(Image.fromarray(cam_u8).resize(pil.size, Image.BILINEAR)) / 255.0
|
||||
colored = (cm.jet(cam_r)[:, :, :3] * 255).astype(np.uint8)
|
||||
return Image.blend(pil.convert("RGB"), Image.fromarray(colored), alpha)
|
||||
|
||||
|
||||
# ── Disc-centred attention helpers ────────────────────────────────────────────
|
||||
|
||||
def _disc_contour_path(pid: int, eye: str, expert: int = 1) -> Path:
|
||||
return CONTOUR_DIR / f"RET{pid:03d}{eye}_disc_exp{expert}.txt"
|
||||
|
||||
|
||||
def _load_disc_mask(pid: int, eye: str, cam_h: int, cam_w: int) -> np.ndarray | None:
|
||||
"""Load expert disc contour, polygon-fill, resize to (cam_h, cam_w)."""
|
||||
from PIL import ImageDraw as _ID
|
||||
p = _disc_contour_path(pid, eye)
|
||||
if not p.exists():
|
||||
return None
|
||||
try:
|
||||
arr = np.loadtxt(str(p), dtype=np.float32)
|
||||
except Exception:
|
||||
return None
|
||||
if arr.ndim == 1:
|
||||
arr = arr.reshape(-1, 2)
|
||||
if arr.shape[0] < 3:
|
||||
return None
|
||||
# Get original image size
|
||||
img_path = get_image_path(pid, eye)
|
||||
try:
|
||||
with Image.open(img_path) as im:
|
||||
orig_w, orig_h = im.size
|
||||
except Exception:
|
||||
return None
|
||||
canvas = Image.new("L", (orig_w, orig_h), 0)
|
||||
_ID.Draw(canvas).polygon([tuple(pt) for pt in arr[:, :2]], fill=1)
|
||||
return np.array(canvas.resize((cam_w, cam_h), Image.NEAREST), dtype=bool)
|
||||
|
||||
|
||||
def _disc_centred_patch(cam: np.ndarray, disc_mask: np.ndarray,
|
||||
span: int = DISC_SPAN, out: int = PATCH_SIZE
|
||||
) -> tuple[np.ndarray | None, float | None]:
|
||||
"""Translate+scale cam so disc centroid is centred; return (patch, disc_r_out)."""
|
||||
if disc_mask is None or disc_mask.sum() == 0:
|
||||
return None, None
|
||||
ys, xs = np.where(disc_mask)
|
||||
cy, cx = ys.mean(), xs.mean()
|
||||
disc_r = float(np.sqrt(disc_mask.sum() / np.pi))
|
||||
half = max(1, int(round(span * disc_r / 2)))
|
||||
h, w = cam.shape
|
||||
y0, y1 = int(round(cy)) - half, int(round(cy)) + half
|
||||
x0, x1 = int(round(cx)) - half, int(round(cx)) + half
|
||||
pt = max(0, -y0); pb = max(0, y1 - h)
|
||||
pl = max(0, -x0); pr = max(0, x1 - w)
|
||||
cam_pad = np.pad(cam, ((pt, pb), (pl, pr)), constant_values=0.0)
|
||||
patch = cam_pad[y0 + pt: y1 + pt, x0 + pl: x1 + pl]
|
||||
patch_out = np.array(
|
||||
Image.fromarray((np.clip(patch, 0, 1) * 255).astype(np.uint8))
|
||||
.resize((out, out), Image.BILINEAR)
|
||||
) / 255.0
|
||||
disc_r_out = out * disc_r / (2 * half)
|
||||
return patch_out.astype(np.float32), disc_r_out
|
||||
|
||||
|
||||
def make_disc_attention_detail(
|
||||
mean_patches: dict,
|
||||
stats_rows: list[dict],
|
||||
out_path: Path,
|
||||
) -> None: # noqa: C901
|
||||
"""
|
||||
2-row (Normal / Glaucoma) × 3-col (correct cam | incorrect cam | disc_frac strip).
|
||||
|
||||
mean_patches: {(cls_name, split): (mean_patch_array, mean_disc_r, count)}
|
||||
stats_rows: list of {true_name, correct, disc_frac} dicts (floats only, no arrays)
|
||||
"""
|
||||
import pandas as pd
|
||||
from matplotlib.patches import Circle
|
||||
|
||||
classes = ["Normal", "Glaucoma"]
|
||||
splits = ["correct", "incorrect"]
|
||||
corr_colors = {"correct": "steelblue", "incorrect": "tomato"}
|
||||
stats = pd.DataFrame(stats_rows)
|
||||
|
||||
# 2 rows (Normal / Glaucoma) × 3 cols (correct cam | incorrect cam | disc_frac strip)
|
||||
fig, axes = plt.subplots(2, 3, figsize=(13, 8),
|
||||
gridspec_kw={"width_ratios": [1, 1, 0.75]})
|
||||
fig.patch.set_facecolor("#f4f4f4")
|
||||
|
||||
rng = np.random.default_rng(42)
|
||||
|
||||
for ri, cls in enumerate(classes):
|
||||
# Col 0 & 1: correct / incorrect mean CAMs
|
||||
for ci, split in enumerate(splits):
|
||||
ax = axes[ri, ci]
|
||||
ax.set_facecolor("#222")
|
||||
key = (cls, split)
|
||||
if key in mean_patches:
|
||||
mp, disc_r_out, count = mean_patches[key]
|
||||
ax.imshow(mp, cmap="jet", vmin=0, vmax=1, origin="upper",
|
||||
extent=[0, PATCH_SIZE, PATCH_SIZE, 0])
|
||||
cx = cy = PATCH_SIZE / 2
|
||||
ax.add_patch(Circle((cx, cy), disc_r_out,
|
||||
fill=False, edgecolor="white",
|
||||
linewidth=2, linestyle="--"))
|
||||
ax.set_title(f"{split.capitalize()} (N={count})", fontsize=9)
|
||||
else:
|
||||
ax.text(0.5, 0.5, "no data", ha="center", va="center",
|
||||
transform=ax.transAxes, fontsize=9, color="grey")
|
||||
ax.set_title(split.capitalize(), fontsize=9)
|
||||
ax.axis("off")
|
||||
|
||||
# Row label on leftmost column
|
||||
axes[ri, 0].set_ylabel(cls, fontsize=11, fontweight="bold", labelpad=8)
|
||||
|
||||
# Col 2: disc_frac strip
|
||||
ax = axes[ri, 2]
|
||||
ax.set_facecolor("#f4f4f4")
|
||||
sub = stats[stats["true_name"] == cls].dropna(subset=["disc_frac"])
|
||||
for xi, split in enumerate(splits):
|
||||
pts = sub[sub["correct"] == (split == "correct")]["disc_frac"].values
|
||||
if len(pts) == 0:
|
||||
continue
|
||||
color = corr_colors[split]
|
||||
jitter = rng.uniform(-0.18, 0.18, size=len(pts))
|
||||
ax.scatter(xi + jitter, pts, color=color, alpha=0.7, s=28, edgecolors="none")
|
||||
ax.hlines(pts.mean(), xi - 0.28, xi + 0.28, colors=color, linewidth=2.5, zorder=5)
|
||||
ax.set_xticks([0, 1])
|
||||
ax.set_xticklabels(["Correct", "Incorrect"], fontsize=9)
|
||||
ax.set_xlim(-0.55, 1.55)
|
||||
ax.set_ylim(0, 1)
|
||||
ax.set_title("Disc fraction", fontsize=9)
|
||||
ax.grid(axis="y", linestyle="--", alpha=0.3)
|
||||
if ri == 0:
|
||||
ax.set_ylabel("Attention mass inside GT disc", fontsize=9)
|
||||
|
||||
fig.suptitle(
|
||||
"Disc-centred GradCAM attention | dashed circle = GT disc boundary\n"
|
||||
"Phase 5, logit_mlp_head, fold 0–4",
|
||||
fontsize=11, fontweight="bold",
|
||||
)
|
||||
fig.tight_layout()
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out_path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {out_path}")
|
||||
|
||||
|
||||
# ── Model loading ─────────────────────────────────────────────────────────────
|
||||
|
||||
def build_model(ckpt_path: Path, device: torch.device):
|
||||
"""Reconstruct SingleEyeHT from checkpoint and load weights."""
|
||||
from types import SimpleNamespace
|
||||
from v3.classes.models import SingleEyeHT
|
||||
sd = torch.load(ckpt_path, map_location="cpu")
|
||||
# ClinicalTower only reads clinical_data.feature_dim at init time
|
||||
cd_in = sd["cd_tower.block0.0.weight"].shape[1]
|
||||
clinical_shim = SimpleNamespace(feature_dim=cd_in)
|
||||
model = SingleEyeHT(
|
||||
backbone=BACKBONE,
|
||||
freeze_ratio=0.0,
|
||||
augment=False,
|
||||
clinical_data=clinical_shim,
|
||||
num_classes=NUM_CLASSES,
|
||||
cd_hidden_dim=CD_HIDDEN,
|
||||
fusion_dim=FUSION_DIM,
|
||||
)
|
||||
model.load_state_dict(sd)
|
||||
model.to(device).eval()
|
||||
return model
|
||||
|
||||
|
||||
# ── Data helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
def build_data_bundle():
|
||||
"""Build the PAPILA DataBundle matching the checkpointed run's feature config."""
|
||||
from v3.classes.papila_builders import build_papila_data
|
||||
import torch as _t
|
||||
# Auto-detect cd_in from the majority of checkpoints (excludes stale reps).
|
||||
import collections as _col
|
||||
all_ckpts = list(CKPT_RUN.glob("rep*/binary/ensemble/fold*/best_single.pt"))
|
||||
if all_ckpts:
|
||||
counts = _col.Counter(
|
||||
_t.load(c, map_location="cpu")["cd_tower.block0.0.weight"].shape[1]
|
||||
for c in all_ckpts
|
||||
)
|
||||
cd_in = counts.most_common(1)[0][0]
|
||||
else:
|
||||
cd_in = 25
|
||||
drop_raw = cd_in <= 21
|
||||
excl = ["Axial_Length"] if cd_in in (21, 23) else []
|
||||
return build_papila_data(
|
||||
image_dir=str(IMAGE_DIR),
|
||||
clinical_dir=str(CLINICAL_DIR),
|
||||
label_col="Diagnosis",
|
||||
cat_cols=["Gender", "Phakic/Pseudophakic"],
|
||||
iop_corr_method="ratio",
|
||||
iop_drop_raw=drop_raw,
|
||||
exclude_cols=excl,
|
||||
)
|
||||
|
||||
|
||||
def get_image_path(pid: int, eye: str) -> Path:
|
||||
return IMAGE_DIR / f"RET{pid:03d}{eye}.jpg"
|
||||
|
||||
|
||||
def build_meta_vector(row, data) -> torch.Tensor:
|
||||
"""Build the training-compatible feature vector via DataBundle.vectorize_row."""
|
||||
vec = data.vectorize_row(row)
|
||||
return torch.tensor(vec, dtype=torch.float32).unsqueeze(0)
|
||||
|
||||
|
||||
# ── Eval transform ────────────────────────────────────────────────────────────
|
||||
|
||||
def get_eval_transform():
|
||||
from torchvision import transforms
|
||||
return transforms.Compose([
|
||||
transforms.Resize(256),
|
||||
transforms.CenterCrop(224),
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize(mean=(0.485, 0.456, 0.406),
|
||||
std=(0.229, 0.224, 0.225)),
|
||||
])
|
||||
|
||||
|
||||
# ── Main loop ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def _discover_checkpoints(ckpt_run: Path) -> list[tuple[int, int, Path]]:
|
||||
"""
|
||||
Scan ckpt_run for all available best_single.pt files.
|
||||
Skips checkpoints whose cd_in doesn't match the majority (to exclude stale reps).
|
||||
Returns sorted list of (rep_idx, fold_idx, ckpt_path).
|
||||
"""
|
||||
import collections
|
||||
candidates = []
|
||||
for rep_dir in sorted(ckpt_run.glob("rep*")):
|
||||
try:
|
||||
rep_idx = int(rep_dir.name.replace("rep", ""))
|
||||
except ValueError:
|
||||
continue
|
||||
for fold_dir in sorted((rep_dir / "binary" / "ensemble").glob("fold[0-9]")):
|
||||
ckpt = fold_dir / "best_single.pt"
|
||||
if ckpt.exists():
|
||||
fold_idx = int(fold_dir.name.replace("fold", ""))
|
||||
cd_in = torch.load(ckpt, map_location="cpu")[
|
||||
"cd_tower.block0.0.weight"
|
||||
].shape[1]
|
||||
candidates.append((rep_idx, fold_idx, ckpt, cd_in))
|
||||
|
||||
if not candidates:
|
||||
return []
|
||||
|
||||
# Use the majority cd_in so stale reps are automatically excluded
|
||||
counts = collections.Counter(c[3] for c in candidates)
|
||||
target_cd_in = counts.most_common(1)[0][0]
|
||||
skipped = sum(1 for c in candidates if c[3] != target_cd_in)
|
||||
if skipped:
|
||||
print(f" [discover] skipping {skipped} checkpoint(s) with cd_in≠{target_cd_in}")
|
||||
|
||||
return [(rep, fold, ckpt) for rep, fold, ckpt, cd in candidates if cd == target_cd_in]
|
||||
|
||||
|
||||
def run(n_grid: int = 16, alpha: float = 0.45, target_class: int | None = None):
|
||||
"""
|
||||
Loop over all available checkpoints in the run directory (all reps × folds).
|
||||
Aggregate CAMs per class, collect overlay grids.
|
||||
"""
|
||||
import pandas as pd
|
||||
from v3.scripts.output_analysis.explainability.fold_patient_ids import (
|
||||
get_test_patient_ids,
|
||||
)
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
print(f"Device: {device}")
|
||||
|
||||
print("Building DataBundle ...")
|
||||
data = build_data_bundle()
|
||||
clinical = data.df
|
||||
print(f" feature_dim={data.feature_dim} rows={len(clinical)}")
|
||||
transform = get_eval_transform()
|
||||
|
||||
checkpoints = _discover_checkpoints(CKPT_RUN)
|
||||
print(f"Found {len(checkpoints)} checkpoint(s) across "
|
||||
f"{len(set(r for r,f,_ in checkpoints))} rep(s)")
|
||||
|
||||
if not checkpoints:
|
||||
print("No checkpoints found — run with --save-checkpoints first.")
|
||||
return
|
||||
|
||||
# ── Incremental accumulators (no full-res arrays kept after each eye) ────────
|
||||
# Mean CAM per class: running sum
|
||||
cam_sum = {0: None, 1: None}
|
||||
cam_count = {0: 0, 1: 0}
|
||||
|
||||
# Overlay grid: keep at most n_grid PIL images per class (capped)
|
||||
overlay_items = {0: [], 1: []}
|
||||
|
||||
# Disc-attention detail: running sum of disc patches (not list of arrays)
|
||||
disc_patch_sum = {} # (cls_name, split) → np.ndarray sum
|
||||
disc_patch_count = {} # (cls_name, split) → int
|
||||
disc_radius_sum = {} # (cls_name, split) → float sum
|
||||
disc_stats_rows = [] # floats only — no arrays
|
||||
|
||||
ckpt_bar = tqdm(checkpoints, desc="Folds", unit="fold")
|
||||
for rep_idx, fold_idx, ckpt_path in ckpt_bar:
|
||||
ckpt_bar.set_postfix(rep=rep_idx, fold=fold_idx)
|
||||
model = build_model(ckpt_path, device)
|
||||
|
||||
# GradCAM target: last ResNet block
|
||||
target_layer = model.img_tower.backbone.layer4[-1]
|
||||
gcam = GradCAM(target_layer)
|
||||
|
||||
pids = get_test_patient_ids(rep_idx, fold_idx, clinical_dir=CLINICAL_DIR)
|
||||
|
||||
for pid in tqdm(pids, desc=f" rep{rep_idx:02d}/fold{fold_idx}", leave=False, unit="pt"):
|
||||
for eye in ("OD", "OS"):
|
||||
img_path = get_image_path(pid, eye)
|
||||
if not img_path.exists():
|
||||
continue
|
||||
|
||||
row = clinical[
|
||||
(clinical["Patient ID"] == pid) & (clinical["eyeID"] == eye)
|
||||
]
|
||||
if len(row) == 0:
|
||||
continue
|
||||
row = row.iloc[0]
|
||||
label = int(row["Diagnosis"])
|
||||
|
||||
pil_orig = Image.open(img_path).convert("RGB")
|
||||
img_t = transform(pil_orig).unsqueeze(0).to(device)
|
||||
meta_t = build_meta_vector(row, data).to(device)
|
||||
|
||||
cam_np, pred = gcam.compute(img_t, meta_t, model,
|
||||
target_class=target_class)
|
||||
|
||||
# Running mean CAM
|
||||
if cam_sum[label] is None:
|
||||
cam_sum[label] = cam_np.copy()
|
||||
else:
|
||||
cam_sum[label] += cam_np
|
||||
cam_count[label] += 1
|
||||
|
||||
# Overlay grid — only keep up to n_grid per class
|
||||
if len(overlay_items[label]) < n_grid:
|
||||
ov = overlay_gradcam(pil_orig, cam_np, alpha=alpha)
|
||||
overlay_items[label].append((ov, pid, eye, pred))
|
||||
|
||||
# Disc-attention: extract patch now, accumulate into running sum
|
||||
h, w = cam_np.shape
|
||||
disc_mask = _load_disc_mask(pid, eye, h, w)
|
||||
disc_frac = None
|
||||
if disc_mask is not None and disc_mask.sum() > 0:
|
||||
disc_frac = float(cam_np[disc_mask].sum() / (cam_np.sum() + 1e-8))
|
||||
|
||||
cls_name = LABEL_NAMES[label]
|
||||
split = "correct" if (pred == label) else "incorrect"
|
||||
key = (cls_name, split)
|
||||
|
||||
patch, disc_r_out = _disc_centred_patch(cam_np, disc_mask)
|
||||
if patch is not None:
|
||||
if key not in disc_patch_sum:
|
||||
disc_patch_sum[key] = patch.copy()
|
||||
disc_patch_count[key] = 1
|
||||
disc_radius_sum[key] = disc_r_out
|
||||
else:
|
||||
disc_patch_sum[key] += patch
|
||||
disc_patch_count[key] += 1
|
||||
disc_radius_sum[key] += disc_r_out
|
||||
|
||||
disc_stats_rows.append({
|
||||
"true_name": cls_name,
|
||||
"correct": pred == label,
|
||||
"disc_frac": disc_frac,
|
||||
})
|
||||
|
||||
# Release per-eye tensors immediately
|
||||
del img_t, meta_t, cam_np
|
||||
if disc_mask is not None:
|
||||
del disc_mask
|
||||
|
||||
gcam.remove()
|
||||
del model
|
||||
torch.cuda.empty_cache() if torch.cuda.is_available() else None
|
||||
|
||||
# Build mean_patches dict for disc detail plot
|
||||
mean_patches = {
|
||||
key: (
|
||||
disc_patch_sum[key] / disc_patch_count[key],
|
||||
disc_radius_sum[key] / disc_patch_count[key],
|
||||
disc_patch_count[key],
|
||||
)
|
||||
for key in disc_patch_sum
|
||||
}
|
||||
|
||||
# ── Save outputs ─────────────────────────────────────────────────────────
|
||||
FIGURES_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for cls in [0, 1]:
|
||||
if cam_count[cls] == 0:
|
||||
continue
|
||||
mean_cam = cam_sum[cls] / cam_count[cls]
|
||||
lo, hi = mean_cam.min(), mean_cam.max()
|
||||
mean_cam = (mean_cam - lo) / (hi - lo + 1e-8)
|
||||
|
||||
fig, ax = plt.subplots(figsize=(5, 5))
|
||||
ax.imshow(mean_cam, cmap="jet", vmin=0, vmax=1)
|
||||
ax.axis("off")
|
||||
ax.set_title(f"Mean GradCAM — {LABEL_NAMES[cls]}\n(n={cam_count[cls]} eyes, fold 0–4)",
|
||||
fontsize=11, fontweight="bold")
|
||||
plt.colorbar(ax.images[0], ax=ax, fraction=0.046, pad=0.04)
|
||||
out = FIGURES_ROOT / f"mean_cam_{LABEL_NAMES[cls].lower()}.png"
|
||||
fig.savefig(out, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {out}")
|
||||
|
||||
# Side-by-side comparison
|
||||
if cam_count[0] > 0 and cam_count[1] > 0:
|
||||
fig, axes = plt.subplots(1, 2, figsize=(10, 5))
|
||||
fig.suptitle("Mean GradCAM — Normal vs Glaucoma (Phase 5, fold 0–4)",
|
||||
fontsize=12, fontweight="bold")
|
||||
for ax, cls in zip(axes, [0, 1]):
|
||||
mean_cam = cam_sum[cls] / cam_count[cls]
|
||||
lo, hi = mean_cam.min(), mean_cam.max()
|
||||
mean_cam = (mean_cam - lo) / (hi - lo + 1e-8)
|
||||
im = ax.imshow(mean_cam, cmap="jet", vmin=0, vmax=1)
|
||||
ax.axis("off")
|
||||
ax.set_title(f"{LABEL_NAMES[cls]} (n={cam_count[cls]})", fontsize=11)
|
||||
plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
|
||||
out = FIGURES_ROOT / "mean_cam_comparison.png"
|
||||
fig.savefig(out, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {out}")
|
||||
|
||||
# Overlay grids
|
||||
for cls in [0, 1]:
|
||||
items = overlay_items[cls]
|
||||
if not items:
|
||||
continue
|
||||
# Sort: misclassified first (more interesting)
|
||||
items.sort(key=lambda x: x[3] == cls) # wrong preds first
|
||||
items = items[:n_grid]
|
||||
ncols = 4
|
||||
nrows = int(np.ceil(len(items) / ncols))
|
||||
fig, axes = plt.subplots(nrows, ncols, figsize=(ncols * 3.2, nrows * 3.2))
|
||||
axes = np.array(axes).reshape(-1)
|
||||
fig.suptitle(f"GradCAM Overlays — {LABEL_NAMES[cls]} (Phase 5)",
|
||||
fontsize=12, fontweight="bold")
|
||||
for i, ax in enumerate(axes):
|
||||
if i < len(items):
|
||||
ov, pid, eye, pred = items[i]
|
||||
ax.imshow(ov)
|
||||
correct = pred == cls
|
||||
col = "#2e7d32" if correct else "#c62828"
|
||||
ax.set_title(f"RET{pid:03d}{eye}\n→ {LABEL_NAMES[pred]}",
|
||||
fontsize=7.5, color=col)
|
||||
ax.axis("off")
|
||||
out = FIGURES_ROOT / f"overlay_grid_{LABEL_NAMES[cls].lower()}.png"
|
||||
fig.tight_layout()
|
||||
fig.savefig(out, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {out}")
|
||||
|
||||
|
||||
# Disc-centred detail plot
|
||||
if disc_patch_sum:
|
||||
make_disc_attention_detail(mean_patches, disc_stats_rows,
|
||||
FIGURES_ROOT / "disc_attention_detail.png")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--n-grid", type=int, default=16,
|
||||
help="Max overlays per class in grid (default 16)")
|
||||
ap.add_argument("--alpha", type=float, default=0.45,
|
||||
help="GradCAM overlay opacity (default 0.45)")
|
||||
ap.add_argument("--target-class", type=int, default=None,
|
||||
help="GradCAM target class (default: predicted class)")
|
||||
args = ap.parse_args()
|
||||
run(n_grid=args.n_grid, alpha=args.alpha, target_class=args.target_class)
|
||||
@@ -0,0 +1,309 @@
|
||||
"""
|
||||
MD permutation feature importance for Phase 5 — logit_mlp_head checkpointed run.
|
||||
|
||||
For each of the 5 fold checkpoints:
|
||||
- loads test images + clinical metadata
|
||||
- caches image features (no grad)
|
||||
- permutes each clinical feature N times and measures AUC drop
|
||||
|
||||
Produces (in figures/explainability/):
|
||||
md_importance_phase5.png — aggregated bar chart across 5 folds
|
||||
md_importance_phase5.csv — mean/std per feature
|
||||
|
||||
Usage:
|
||||
python -m v3.scripts.output_analysis.explainability.permutation_importance_phase5
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from sklearn.metrics import roc_auc_score
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
CKPT_RUN = REPO_ROOT / "v3" / "results" / "phase5" / "logit_mlp_head_ckpt"
|
||||
FIGURES_ROOT = REPO_ROOT / "v3" / "figures" / "explainability"
|
||||
CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
|
||||
IMAGE_DIR = REPO_ROOT / "Papila" / "FundusImages"
|
||||
|
||||
BACKBONE = "resnet50"
|
||||
NUM_CLASSES = 2
|
||||
CD_HIDDEN = 128
|
||||
FUSION_DIM = 256
|
||||
N_PERMUTATIONS = 30
|
||||
SEED = 0
|
||||
|
||||
|
||||
# ── Model ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def build_model(ckpt_path: Path, device: torch.device):
|
||||
from v3.classes.models import SingleEyeHT
|
||||
sd = torch.load(ckpt_path, map_location="cpu")
|
||||
cd_in = sd["cd_tower.block0.0.weight"].shape[1]
|
||||
model = SingleEyeHT(
|
||||
backbone=BACKBONE, freeze_ratio=0.0, augment=False,
|
||||
clinical_data=SimpleNamespace(feature_dim=cd_in),
|
||||
num_classes=NUM_CLASSES, cd_hidden_dim=CD_HIDDEN, fusion_dim=FUSION_DIM,
|
||||
)
|
||||
model.load_state_dict(sd)
|
||||
model.to(device).eval()
|
||||
return model
|
||||
|
||||
|
||||
# ── Data ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def build_data_bundle():
|
||||
from v3.classes.papila_builders import build_papila_data
|
||||
# Infer settings from available checkpoint to stay compatible.
|
||||
# Once the 10x5 run (--iop-drop-raw --exclude-cols Axial_Length) completes,
|
||||
# these will automatically match (feature_dim will drop from 25 → 21).
|
||||
ckpt = next(CKPT_RUN.glob("rep*/binary/ensemble/fold*/best_single.pt"), None)
|
||||
import torch as _t
|
||||
cd_in = _t.load(ckpt, map_location="cpu")["cd_tower.block0.0.weight"].shape[1] if ckpt else 25
|
||||
# cd_in=25 → old run (no iop_drop_raw, no excl); cd_in=21 → new run
|
||||
drop_raw = cd_in <= 21
|
||||
excl = ["Axial_Length"] if cd_in in (21, 23) else []
|
||||
return build_papila_data(
|
||||
image_dir=str(IMAGE_DIR), clinical_dir=str(CLINICAL_DIR),
|
||||
label_col="Diagnosis", cat_cols=["Gender", "Phakic/Pseudophakic"],
|
||||
iop_corr_method="ratio", iop_drop_raw=drop_raw, exclude_cols=excl,
|
||||
)
|
||||
|
||||
|
||||
def get_eval_transform():
|
||||
from torchvision import transforms
|
||||
return transforms.Compose([
|
||||
transforms.Resize(256), transforms.CenterCrop(224),
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
|
||||
])
|
||||
|
||||
|
||||
def get_image_path(pid: int, eye: str) -> Path:
|
||||
return IMAGE_DIR / f"RET{pid:03d}{eye}.jpg"
|
||||
|
||||
|
||||
# ── Feature index map ─────────────────────────────────────────────────────────
|
||||
|
||||
def build_feature_index_map(data) -> dict[str, list[int]]:
|
||||
"""
|
||||
Map feature name → list of dimension indices in the vectorize_row output.
|
||||
Layout: [scalars (min-max scaled)] + [cat one-hots] + [scalar missing flags]
|
||||
"""
|
||||
n_scalar = len(data.scalar_cols)
|
||||
cat_expanded = sum(len(m) for m in data.cat_maps.values())
|
||||
feat_map: dict[str, list[int]] = {}
|
||||
|
||||
# Scalar: value dim + missing flag dim
|
||||
for i, col in enumerate(data.scalar_cols):
|
||||
feat_map[col] = [i, n_scalar + cat_expanded + i]
|
||||
|
||||
# Categorical: whole one-hot block
|
||||
cat_offset = n_scalar
|
||||
for col in data.cat_cols:
|
||||
n = len(data.cat_maps[col])
|
||||
feat_map[col] = list(range(cat_offset, cat_offset + n))
|
||||
cat_offset += n
|
||||
|
||||
return feat_map
|
||||
|
||||
|
||||
# ── Per-fold importance ───────────────────────────────────────────────────────
|
||||
|
||||
def run_fold(rep_idx: int, fold_idx: int, model, data, device: torch.device,
|
||||
n_permutations: int, seed: int) -> dict[str, tuple[float, float]]:
|
||||
"""
|
||||
Returns {feature_name: (mean_auc_drop, std_auc_drop)}.
|
||||
"""
|
||||
from v3.scripts.output_analysis.explainability.fold_patient_ids import (
|
||||
get_test_patient_ids,
|
||||
)
|
||||
transform = get_eval_transform()
|
||||
clinical = data.df
|
||||
|
||||
pids = get_test_patient_ids(rep_idx, fold_idx, clinical_dir=CLINICAL_DIR)
|
||||
|
||||
# Cache image features + build meta tensors + labels
|
||||
img_feats_list, meta_list, label_list = [], [], []
|
||||
model.eval()
|
||||
with torch.no_grad():
|
||||
for pid in pids:
|
||||
for eye in ("OD", "OS"):
|
||||
img_path = get_image_path(pid, eye)
|
||||
if not img_path.exists():
|
||||
continue
|
||||
row = clinical[
|
||||
(clinical["Patient ID"] == pid) & (clinical["eyeID"] == eye)
|
||||
]
|
||||
if len(row) == 0:
|
||||
continue
|
||||
row = row.iloc[0]
|
||||
label = int(row["Diagnosis"])
|
||||
|
||||
from PIL import Image
|
||||
pil = Image.open(img_path).convert("RGB")
|
||||
img_t = transform(pil).unsqueeze(0).to(device)
|
||||
feats = model.img_tower(img_t) # [1, img_dim]
|
||||
meta_vec = torch.tensor(data.vectorize_row(row),
|
||||
dtype=torch.float32).unsqueeze(0)
|
||||
|
||||
img_feats_list.append(feats.cpu())
|
||||
meta_list.append(meta_vec)
|
||||
label_list.append(label)
|
||||
|
||||
if not label_list or len(set(label_list)) < 2:
|
||||
print(f" fold{fold_idx}: insufficient data, skipping.")
|
||||
return {}
|
||||
|
||||
img_feats = torch.cat(img_feats_list).to(device) # [N, img_dim]
|
||||
meta_all = torch.cat(meta_list) # [N, feat_dim] on CPU
|
||||
y_true = np.array(label_list)
|
||||
|
||||
# Baseline AUC
|
||||
with torch.no_grad():
|
||||
md_feats = model.cd_tower(meta_all.to(device))
|
||||
out_f, _, _ = model.bridge(img_feats, md_feats)
|
||||
probs_base = F.softmax(out_f, dim=1)[:, 1].cpu().numpy()
|
||||
baseline_auc = roc_auc_score(y_true, probs_base)
|
||||
print(f" fold{fold_idx}: baseline AUC={baseline_auc:.4f} N={len(y_true)}")
|
||||
|
||||
feat_map = build_feature_index_map(data)
|
||||
rng = np.random.default_rng(seed + fold_idx)
|
||||
results: dict[str, tuple[float, float]] = {}
|
||||
|
||||
for feat_name, dims in feat_map.items():
|
||||
drops = []
|
||||
for _ in range(n_permutations):
|
||||
meta_perm = meta_all.clone()
|
||||
perm_idx = rng.permutation(len(meta_perm))
|
||||
meta_perm[:, dims] = meta_perm[perm_idx][:, dims]
|
||||
with torch.no_grad():
|
||||
md_p = model.cd_tower(meta_perm.to(device))
|
||||
out_p, _, _ = model.bridge(img_feats, md_p)
|
||||
probs_p = F.softmax(out_p, dim=1)[:, 1].cpu().numpy()
|
||||
try:
|
||||
drops.append(baseline_auc - roc_auc_score(y_true, probs_p))
|
||||
except Exception:
|
||||
pass
|
||||
if drops:
|
||||
results[feat_name] = (float(np.mean(drops)), float(np.std(drops)))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# ── Aggregate and plot ────────────────────────────────────────────────────────
|
||||
|
||||
def plot_importance(all_results: list[dict], out_png: Path, out_csv: Path) -> None:
|
||||
# Aggregate across folds
|
||||
all_feats = sorted({f for r in all_results for f in r})
|
||||
agg = {}
|
||||
for feat in all_feats:
|
||||
vals = [r[feat][0] for r in all_results if feat in r]
|
||||
if vals:
|
||||
agg[feat] = (float(np.mean(vals)), float(np.std(vals)))
|
||||
|
||||
# Sort by mean importance descending
|
||||
sorted_feats = sorted(agg, key=lambda f: agg[f][0], reverse=True)
|
||||
names = sorted_feats
|
||||
imps = [agg[f][0] for f in names]
|
||||
stds = [agg[f][1] for f in names]
|
||||
colors = ["#e05c5c" if v >= 0 else "#5c9ee0" for v in imps]
|
||||
|
||||
fig, ax = plt.subplots(figsize=(9, max(4, len(names) * 0.45 + 1.5)))
|
||||
y_pos = np.arange(len(names))
|
||||
ax.barh(y_pos, imps, xerr=stds, color=colors, ecolor="grey", capsize=3, height=0.6)
|
||||
ax.set_yticks(y_pos)
|
||||
ax.set_yticklabels(names, fontsize=9)
|
||||
ax.invert_yaxis()
|
||||
ax.axvline(0, color="black", linewidth=0.8)
|
||||
ax.set_xlabel("Mean AUC drop (baseline − permuted)", fontsize=10)
|
||||
n_reps = len(set(r for r in range(len(all_results)))) # placeholder
|
||||
ax.set_title(
|
||||
f"MD Tower — Permutation Feature Importance\n"
|
||||
f"Phase 5 logit_mlp_head_ckpt ({len(all_results)} folds, "
|
||||
f"error bars = std across folds)",
|
||||
fontsize=11,
|
||||
)
|
||||
fig.tight_layout()
|
||||
out_png.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out_png, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {out_png}")
|
||||
|
||||
with open(out_csv, "w", newline="") as f:
|
||||
w = csv.DictWriter(f, fieldnames=["feature", "mean_importance", "std_importance"])
|
||||
w.writeheader()
|
||||
for feat in sorted_feats:
|
||||
w.writerow({"feature": feat,
|
||||
"mean_importance": agg[feat][0],
|
||||
"std_importance": agg[feat][1]})
|
||||
print(f"Saved: {out_csv}")
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _discover_checkpoints(ckpt_run: Path) -> list[tuple[int, int, Path]]:
|
||||
found = []
|
||||
for rep_dir in sorted(ckpt_run.glob("rep*")):
|
||||
try:
|
||||
rep_idx = int(rep_dir.name.replace("rep", ""))
|
||||
except ValueError:
|
||||
continue
|
||||
for fold_dir in sorted((rep_dir / "binary" / "ensemble").glob("fold[0-9]")):
|
||||
ckpt = fold_dir / "best_single.pt"
|
||||
if ckpt.exists():
|
||||
found.append((rep_idx, int(fold_dir.name.replace("fold", "")), ckpt))
|
||||
return found
|
||||
|
||||
|
||||
def main(n_permutations: int = N_PERMUTATIONS, seed: int = SEED):
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
print(f"Device: {device}")
|
||||
|
||||
print("Building DataBundle ...")
|
||||
data = build_data_bundle()
|
||||
print(f" feature_dim={data.feature_dim}")
|
||||
|
||||
checkpoints = _discover_checkpoints(CKPT_RUN)
|
||||
print(f"Found {len(checkpoints)} checkpoint(s) across "
|
||||
f"{len(set(r for r,f,_ in checkpoints))} rep(s)")
|
||||
|
||||
if not checkpoints:
|
||||
print("No checkpoints found.")
|
||||
return
|
||||
|
||||
all_results = []
|
||||
for rep_idx, fold_idx, ckpt in checkpoints:
|
||||
print(f"\n── rep{rep_idx:02d} fold{fold_idx} ──")
|
||||
model = build_model(ckpt, device)
|
||||
result = run_fold(rep_idx, fold_idx, model, data, device, n_permutations, seed)
|
||||
if result:
|
||||
all_results.append(result)
|
||||
del model
|
||||
|
||||
if not all_results:
|
||||
print("No results — nothing to plot.")
|
||||
return
|
||||
|
||||
plot_importance(
|
||||
all_results,
|
||||
FIGURES_ROOT / "md_importance_phase5.png",
|
||||
FIGURES_ROOT / "md_importance_phase5.csv",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--n-permutations", type=int, default=N_PERMUTATIONS)
|
||||
ap.add_argument("--seed", type=int, default=SEED)
|
||||
args = ap.parse_args()
|
||||
main(n_permutations=args.n_permutations, seed=args.seed)
|
||||
@@ -0,0 +1,693 @@
|
||||
"""
|
||||
Publication-quality architecture diagrams for HyperTower.
|
||||
|
||||
Generates:
|
||||
architecture_single_tower.png — single-eye image-only tower
|
||||
architecture_hypertower.png — single-eye image + clinical fusion
|
||||
architecture_ensemble.png — bilateral ensemble (two HyperTowers + average)
|
||||
architecture_fused_head.png — bilateral ensemble + learned head
|
||||
|
||||
Usage:
|
||||
python -m v3.scripts.output_analysis.plot_architecture
|
||||
python -m v3.scripts.output_analysis.plot_architecture --out figures/
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.patches import FancyBboxPatch, FancyArrowPatch
|
||||
import matplotlib.patheffects as pe
|
||||
|
||||
# ── Colour palette ────────────────────────────────────────────────────────────
|
||||
C_IMG = "#4e8d3a" # green — image / CNN
|
||||
C_MD = "#4c72b0" # blue — clinical / MLP
|
||||
C_BRIDGE = "#c44e52" # red — bridge / fusion
|
||||
C_EMB = "#2a9d8f" # teal — embedding vectors (z)
|
||||
C_OUT = "#8c6bb1" # purple — output nodes
|
||||
C_HEAD = "#d4a017" # gold — learned head / average
|
||||
C_INPUT = "#a0a0a0" # grey — raw input nodes
|
||||
C_BG = "#e8e8e8"
|
||||
C_ARROW = "#444444"
|
||||
FONT = "DejaVu Sans"
|
||||
|
||||
|
||||
# ── Low-level primitives ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _box(
|
||||
ax,
|
||||
cx,
|
||||
cy,
|
||||
w,
|
||||
h,
|
||||
color,
|
||||
text="",
|
||||
fontsize=9,
|
||||
text_color="white",
|
||||
bold=False,
|
||||
alpha=0.92,
|
||||
radius=0.12,
|
||||
lw=1.5,
|
||||
):
|
||||
"""Rounded rectangle centered at (cx, cy)."""
|
||||
patch = FancyBboxPatch(
|
||||
(cx - w / 2, cy - h / 2),
|
||||
w,
|
||||
h,
|
||||
boxstyle=f"round,pad=0,rounding_size={radius}",
|
||||
facecolor=color,
|
||||
edgecolor="white",
|
||||
linewidth=lw,
|
||||
alpha=alpha,
|
||||
zorder=3,
|
||||
transform=ax.transData,
|
||||
)
|
||||
ax.add_patch(patch)
|
||||
if text:
|
||||
ax.text(
|
||||
cx,
|
||||
cy,
|
||||
text,
|
||||
ha="center",
|
||||
va="center",
|
||||
fontsize=fontsize,
|
||||
color=text_color,
|
||||
fontweight="bold" if bold else "normal",
|
||||
fontfamily=FONT,
|
||||
zorder=4,
|
||||
)
|
||||
return patch
|
||||
|
||||
|
||||
def _arrow(ax, x0, y0, x1, y1, lw=1.6, color=C_ARROW, style="->", rad=0.0):
|
||||
ax.annotate(
|
||||
"",
|
||||
xy=(x1, y1),
|
||||
xytext=(x0, y0),
|
||||
arrowprops=dict(
|
||||
arrowstyle=style,
|
||||
color=color,
|
||||
lw=lw,
|
||||
connectionstyle=f"arc3,rad={rad}",
|
||||
),
|
||||
zorder=2,
|
||||
)
|
||||
|
||||
|
||||
def _text(
|
||||
ax, x, y, s, fontsize=8, color="#333333", ha="center", va="center", bold=False
|
||||
):
|
||||
ax.text(
|
||||
x,
|
||||
y,
|
||||
s,
|
||||
ha=ha,
|
||||
va=va,
|
||||
fontsize=fontsize,
|
||||
color=color,
|
||||
fontfamily=FONT,
|
||||
fontweight="bold" if bold else "normal",
|
||||
zorder=5,
|
||||
)
|
||||
|
||||
|
||||
def _bracket(
|
||||
ax,
|
||||
x,
|
||||
y0,
|
||||
y1,
|
||||
text="",
|
||||
fontsize=8.5,
|
||||
color="#888888",
|
||||
pad=0.15,
|
||||
lw=1.4,
|
||||
badge_color=None,
|
||||
):
|
||||
"""Vertical C-bracket on the right side.
|
||||
If badge_color is set, the label is drawn as white text on a filled badge."""
|
||||
mid = (y0 + y1) / 2
|
||||
ax.plot(
|
||||
[x, x + pad, x + pad, x],
|
||||
[y1, y1, y0, y0],
|
||||
color=color,
|
||||
lw=lw,
|
||||
solid_capstyle="round",
|
||||
zorder=2,
|
||||
)
|
||||
if text:
|
||||
if badge_color:
|
||||
ax.text(
|
||||
x + pad * 1.4,
|
||||
mid,
|
||||
text,
|
||||
ha="left",
|
||||
va="center",
|
||||
fontsize=fontsize,
|
||||
color="white",
|
||||
fontfamily=FONT,
|
||||
fontweight="bold",
|
||||
zorder=6,
|
||||
bbox=dict(
|
||||
facecolor=badge_color,
|
||||
edgecolor="none",
|
||||
pad=3.5,
|
||||
boxstyle="round,pad=0.3",
|
||||
),
|
||||
)
|
||||
else:
|
||||
ax.text(
|
||||
x + pad * 1.4,
|
||||
mid,
|
||||
text,
|
||||
ha="left",
|
||||
va="center",
|
||||
fontsize=fontsize,
|
||||
color=color,
|
||||
fontfamily=FONT,
|
||||
style="italic",
|
||||
)
|
||||
|
||||
|
||||
def _setup(fig, ax, w, h, title):
|
||||
ax.set_xlim(0, w)
|
||||
ax.set_ylim(0, h)
|
||||
ax.axis("off")
|
||||
ax.set_facecolor(C_BG)
|
||||
fig.patch.set_facecolor(C_BG)
|
||||
if title:
|
||||
ax.set_title(
|
||||
title,
|
||||
fontsize=12,
|
||||
fontweight="bold",
|
||||
fontfamily=FONT,
|
||||
pad=10,
|
||||
color="#222222",
|
||||
)
|
||||
|
||||
|
||||
# ── Reusable sub-blocks ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _draw_cnn_block(ax, x_center, y, w=2.0, h=0.75):
|
||||
"""Three-layer CNN block with labels: Conv Layers → Conv Layers → GAP."""
|
||||
labels = ["Conv\nLayers", "Conv\nLayers", "GAP"]
|
||||
sub_w = [w * 0.42, w * 0.30, w * 0.22]
|
||||
sub_h = [h, h * 0.82, h * 0.65]
|
||||
alphas = [0.82, 0.74, 0.66]
|
||||
fsizes = [8.0, 7.5, 7.5]
|
||||
gap = (w - sum(sub_w)) / 2
|
||||
xs = [
|
||||
x_center - w / 2 + sub_w[0] / 2,
|
||||
x_center - w / 2 + sub_w[0] + gap + sub_w[1] / 2,
|
||||
x_center - w / 2 + sub_w[0] + gap + sub_w[1] + gap + sub_w[2] / 2,
|
||||
]
|
||||
for i, (sx, sw, sh, lbl, alp, fs) in enumerate(
|
||||
zip(xs, sub_w, sub_h, labels, alphas, fsizes)
|
||||
):
|
||||
_box(ax, sx, y, sw, sh, C_IMG, lbl, fontsize=fs, alpha=alp, radius=0.08)
|
||||
if i < 2:
|
||||
_arrow(
|
||||
ax, sx + sw / 2, y, xs[i + 1] - sub_w[i + 1] / 2, y, lw=1.2, style="-|>"
|
||||
)
|
||||
return xs[-1] + sub_w[-1] / 2
|
||||
|
||||
|
||||
def _draw_mlp_block(ax, x_center, y, w=1.4, h=0.65):
|
||||
"""Two-layer MLP block: FC(128) → FC(128) (hidden_dim=128 both layers)."""
|
||||
labels = ["FC\n(128)", "FC\n(128)"]
|
||||
w0, w1 = w * 0.55, w * 0.45
|
||||
gap = w - w0 - w1
|
||||
x0 = x_center - w / 2 + w0 / 2
|
||||
x1 = x0 + w0 / 2 + gap + w1 / 2
|
||||
_box(ax, x0, y, w0, h, C_MD, labels[0], fontsize=8.0, alpha=0.82, radius=0.08)
|
||||
_arrow(ax, x0 + w0 / 2, y, x1 - w1 / 2, y, lw=1.2, style="-|>")
|
||||
_box(
|
||||
ax, x1, y, w1, h * 0.88, C_MD, labels[1], fontsize=7.5, alpha=0.72, radius=0.08
|
||||
)
|
||||
return x1 + w1 / 2
|
||||
|
||||
|
||||
def _draw_embedding(ax, x, y, w=0.40, h=0.75, label="z\n(emb)"):
|
||||
_box(ax, x + w / 2, y, w, h, C_EMB, label, fontsize=8, bold=True, radius=0.08)
|
||||
return x + w
|
||||
|
||||
|
||||
def _draw_output(ax, x, y, dy=0.45, classes=("Glaucoma", "Normal")):
|
||||
"""Stacked output class boxes, connected from (x, y) via arrows."""
|
||||
n = len(classes)
|
||||
bw = 1.10
|
||||
bh = 0.38
|
||||
gap = 0.08
|
||||
total = n * bh + (n - 1) * gap
|
||||
y_top = y + total / 2 - bh / 2
|
||||
|
||||
for i, cls in enumerate(classes):
|
||||
cy = y_top - i * (bh + gap)
|
||||
_box(ax, x + bw / 2, cy, bw, bh, C_OUT, cls, fontsize=8.5, radius=0.08)
|
||||
_arrow(ax, x, y, x, cy, lw=1.1, style="-|>", rad=0.0)
|
||||
|
||||
_text(ax, x + bw / 2, y - total / 2 - 0.20, "Softmax", fontsize=7.5, color=C_OUT)
|
||||
|
||||
|
||||
def _draw_compact_ht(ax, x_left, y_img, y_md, eye_label):
|
||||
"""Compact HyperTower block: Image+Clinical boxes → Bridge.
|
||||
Returns (x_right_of_bridge, y_bridge_center).
|
||||
"""
|
||||
bw_img = 1.40
|
||||
bh_img = 0.72
|
||||
bw_md = 1.20
|
||||
bh_md = 0.62
|
||||
bw_br = 0.72
|
||||
cy_br = (y_img + y_md) / 2
|
||||
bh_br = abs(y_img - y_md) * 0.60
|
||||
|
||||
# Image box: CNN Backbone
|
||||
_box(
|
||||
ax,
|
||||
x_left + bw_img / 2,
|
||||
y_img,
|
||||
bw_img,
|
||||
bh_img,
|
||||
C_IMG,
|
||||
f"{eye_label}\nCNN Backbone",
|
||||
fontsize=8.5,
|
||||
radius=0.08,
|
||||
)
|
||||
# MD box: Clinical MLP
|
||||
_box(
|
||||
ax,
|
||||
x_left + bw_md / 2,
|
||||
y_md,
|
||||
bw_md,
|
||||
bh_md,
|
||||
C_MD,
|
||||
f"{eye_label}\nClinical MLP",
|
||||
fontsize=8.5,
|
||||
radius=0.08,
|
||||
)
|
||||
|
||||
# Arrows to bridge
|
||||
br_x = x_left + max(bw_img, bw_md) + 0.60
|
||||
_arrow(ax, x_left + bw_img, y_img, br_x - bw_br / 2, cy_br, lw=1.2, style="-|>")
|
||||
_arrow(ax, x_left + bw_md, y_md, br_x - bw_br / 2, cy_br, lw=1.2, style="-|>")
|
||||
|
||||
# Bridge label kept simple — detail lives in the hypertower diagram
|
||||
_box(
|
||||
ax,
|
||||
br_x,
|
||||
cy_br,
|
||||
bw_br,
|
||||
max(bh_br, 0.70),
|
||||
C_BRIDGE,
|
||||
"Bridge",
|
||||
fontsize=8.0,
|
||||
radius=0.08,
|
||||
)
|
||||
|
||||
return br_x + bw_br / 2, cy_br
|
||||
|
||||
|
||||
# ── Figure 1: Single Tower ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def make_single_tower(out_dir: Path):
|
||||
W, H = 9.0, 3.2
|
||||
fig, ax = plt.subplots(figsize=(W, H))
|
||||
_setup(fig, ax, W, H, "Single Tower")
|
||||
|
||||
cy = H / 2
|
||||
|
||||
# Input
|
||||
_box(
|
||||
ax,
|
||||
0.75,
|
||||
cy,
|
||||
0.95,
|
||||
0.60,
|
||||
C_INPUT,
|
||||
"Fundus\nImage",
|
||||
fontsize=8.5,
|
||||
radius=0.08,
|
||||
alpha=0.75,
|
||||
text_color="#333",
|
||||
)
|
||||
_arrow(ax, 1.22, cy, 1.60, cy)
|
||||
|
||||
# CNN Backbone
|
||||
cnn_x_right = _draw_cnn_block(ax, x_center=3.10, y=cy, w=2.80, h=0.78)
|
||||
_text(ax, 3.10, cy - 0.68, "CNN Backbone", fontsize=8.5, color=C_IMG, bold=True)
|
||||
_arrow(ax, 1.60, cy, 1.73, cy, lw=1.4, style="-|>")
|
||||
|
||||
# Embedding
|
||||
emb_x_right = _draw_embedding(ax, x=cnn_x_right + 0.28, y=cy, w=0.48, h=0.78)
|
||||
_arrow(ax, cnn_x_right, cy, cnn_x_right + 0.28, cy, lw=1.4, style="-|>")
|
||||
|
||||
# Classifier
|
||||
_arrow(ax, emb_x_right, cy, emb_x_right + 0.25, cy, lw=1.4, style="-|>")
|
||||
_draw_output(ax, emb_x_right + 0.25, cy)
|
||||
|
||||
path = out_dir / "architecture_single_tower.png"
|
||||
fig.savefig(path, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {path}")
|
||||
|
||||
|
||||
# ── Figure 2: HyperTower (single eye) ────────────────────────────────────────
|
||||
|
||||
|
||||
def make_hypertower(out_dir: Path):
|
||||
W, H = 11.0, 5.5
|
||||
fig, ax = plt.subplots(figsize=(W, H))
|
||||
_setup(fig, ax, W, H, "HyperTower — Single Eye")
|
||||
|
||||
y_img = 3.70
|
||||
y_md = 1.60
|
||||
|
||||
# ── Image tower ────────────────────────────────────────────────
|
||||
_box(
|
||||
ax,
|
||||
0.80,
|
||||
y_img,
|
||||
1.00,
|
||||
0.60,
|
||||
C_INPUT,
|
||||
"Fundus\nImage",
|
||||
fontsize=8.5,
|
||||
radius=0.08,
|
||||
alpha=0.75,
|
||||
text_color="#333",
|
||||
)
|
||||
_arrow(ax, 1.30, y_img, 1.85, y_img)
|
||||
cnn_x_r = _draw_cnn_block(ax, x_center=3.50, y=y_img, w=2.80, h=0.75)
|
||||
_text(ax, 3.50, y_img - 0.65, "CNN Backbone", fontsize=8, color=C_IMG, bold=True)
|
||||
_arrow(ax, 1.85, y_img, 1.98, y_img, lw=1.4, style="-|>")
|
||||
|
||||
emb_img_x = _draw_embedding(ax, x=cnn_x_r + 0.30, y=y_img, w=0.65, h=0.75)
|
||||
_arrow(ax, cnn_x_r, y_img, cnn_x_r + 0.30, y_img, lw=1.4, style="-|>")
|
||||
_text(
|
||||
ax,
|
||||
(1.30 + emb_img_x) / 2,
|
||||
y_img + 0.65,
|
||||
"Image Tower",
|
||||
fontsize=9,
|
||||
color=C_IMG,
|
||||
bold=True,
|
||||
)
|
||||
|
||||
# ── Clinical tower ─────────────────────────────────────────────
|
||||
_box(
|
||||
ax,
|
||||
0.80,
|
||||
y_md,
|
||||
1.00,
|
||||
0.55,
|
||||
C_INPUT,
|
||||
"Clinical\nData",
|
||||
fontsize=8.5,
|
||||
radius=0.08,
|
||||
alpha=0.75,
|
||||
text_color="#333",
|
||||
)
|
||||
_arrow(ax, 1.30, y_md, 1.65, y_md)
|
||||
mlp_x_r = _draw_mlp_block(ax, x_center=2.90, y=y_md, w=1.60, h=0.65)
|
||||
_arrow(ax, 1.65, y_md, 1.74, y_md, lw=1.4, style="-|>")
|
||||
|
||||
emb_md_x = _draw_embedding(ax, x=mlp_x_r + 0.30, y=y_md, w=0.65, h=0.65)
|
||||
_arrow(ax, mlp_x_r, y_md, mlp_x_r + 0.30, y_md, lw=1.4, style="-|>")
|
||||
_text(
|
||||
ax,
|
||||
(1.30 + emb_md_x) / 2,
|
||||
y_md - 0.60,
|
||||
"Clinical Tower",
|
||||
fontsize=9,
|
||||
color=C_MD,
|
||||
bold=True,
|
||||
)
|
||||
|
||||
# ── Bridge ─────────────────────────────────────────────────────
|
||||
br_x = max(emb_img_x, emb_md_x) + 0.80
|
||||
cy_br = (y_img + y_md) / 2
|
||||
bh_br = abs(y_img - y_md) * 0.55
|
||||
|
||||
_arrow(ax, emb_img_x, y_img, br_x - 0.40, cy_br, lw=1.4, style="-|>")
|
||||
_arrow(ax, emb_md_x, y_md, br_x - 0.40, cy_br, lw=1.4, style="-|>")
|
||||
_box(
|
||||
ax,
|
||||
br_x,
|
||||
cy_br,
|
||||
1.40,
|
||||
max(bh_br, 1.35),
|
||||
C_BRIDGE,
|
||||
"Bridge\nFC(img→256)\nFC(md→256)\n⊙ Hadamard\n→ ReLU→FC(2)",
|
||||
fontsize=8,
|
||||
radius=0.10,
|
||||
)
|
||||
|
||||
# ── Output ─────────────────────────────────────────────────────
|
||||
out_x = br_x + 0.65 + 0.40
|
||||
_arrow(ax, br_x + 0.65, cy_br, out_x, cy_br, lw=1.4, style="-|>")
|
||||
_draw_output(ax, out_x, cy_br)
|
||||
|
||||
# ── Bracket (right of output nodes; output bw=1.10 so right edge = out_x+1.10)
|
||||
_bracket(
|
||||
ax,
|
||||
x=out_x + 1.25,
|
||||
y0=y_md - 0.50,
|
||||
y1=y_img + 0.50,
|
||||
text="HyperTower",
|
||||
fontsize=9,
|
||||
pad=0.22,
|
||||
color="#555",
|
||||
badge_color="#555",
|
||||
)
|
||||
|
||||
path = out_dir / "architecture_hypertower.png"
|
||||
fig.savefig(path, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {path}")
|
||||
|
||||
|
||||
# ── Figure 3: Bilateral Ensemble ─────────────────────────────────────────────
|
||||
|
||||
|
||||
def make_ensemble(out_dir: Path):
|
||||
W, H = 10.5, 7.5
|
||||
fig, ax = plt.subplots(figsize=(W, H))
|
||||
_setup(fig, ax, W, H, "Bilateral Ensemble HyperTower")
|
||||
|
||||
x_left = 3.0
|
||||
inp_cx = 1.85
|
||||
inp_w = 0.90
|
||||
inp_h = 0.55
|
||||
|
||||
# OD (top)
|
||||
od_y_img, od_y_md = 5.90, 4.60
|
||||
br_od_x, cy_od = _draw_compact_ht(
|
||||
ax, x_left=x_left, y_img=od_y_img, y_md=od_y_md, eye_label="OD"
|
||||
)
|
||||
_text(ax, 0.45, (od_y_img + od_y_md) / 2, "OD\n(Right Eye)",
|
||||
fontsize=9, color="#444", bold=True)
|
||||
_box(ax, inp_cx, od_y_img, inp_w, inp_h, C_INPUT, "Fundus\nImage",
|
||||
fontsize=8, radius=0.08, alpha=0.75, text_color="#333")
|
||||
_box(ax, inp_cx, od_y_md, inp_w, inp_h, C_INPUT, "Clinical\nData",
|
||||
fontsize=8, radius=0.08, alpha=0.75, text_color="#333")
|
||||
_arrow(ax, inp_cx + inp_w / 2, od_y_img, x_left, od_y_img, lw=1.2, style="-|>")
|
||||
_arrow(ax, inp_cx + inp_w / 2, od_y_md, x_left, od_y_md, lw=1.2, style="-|>")
|
||||
|
||||
# OS (bottom)
|
||||
os_y_img, os_y_md = 2.80, 1.50
|
||||
br_os_x, cy_os = _draw_compact_ht(
|
||||
ax, x_left=x_left, y_img=os_y_img, y_md=os_y_md, eye_label="OS"
|
||||
)
|
||||
_text(ax, 0.45, (os_y_img + os_y_md) / 2, "OS\n(Left Eye)",
|
||||
fontsize=9, color="#444", bold=True)
|
||||
_box(ax, inp_cx, os_y_img, inp_w, inp_h, C_INPUT, "Fundus\nImage",
|
||||
fontsize=8, radius=0.08, alpha=0.75, text_color="#333")
|
||||
_box(ax, inp_cx, os_y_md, inp_w, inp_h, C_INPUT, "Clinical\nData",
|
||||
fontsize=8, radius=0.08, alpha=0.75, text_color="#333")
|
||||
_arrow(ax, inp_cx + inp_w / 2, os_y_img, x_left, os_y_img, lw=1.2, style="-|>")
|
||||
_arrow(ax, inp_cx + inp_w / 2, os_y_md, x_left, os_y_md, lw=1.2, style="-|>")
|
||||
|
||||
# Average node
|
||||
avg_x = max(br_od_x, br_os_x) + 1.20
|
||||
avg_y = (cy_od + cy_os) / 2
|
||||
avg_size = 0.90
|
||||
|
||||
_arrow(ax, br_od_x, cy_od, avg_x - avg_size / 2, avg_y, lw=1.4, style="-|>")
|
||||
_arrow(ax, br_os_x, cy_os, avg_x - avg_size / 2, avg_y, lw=1.4, style="-|>")
|
||||
_box(
|
||||
ax,
|
||||
avg_x,
|
||||
avg_y,
|
||||
avg_size,
|
||||
avg_size,
|
||||
C_HEAD,
|
||||
"Average",
|
||||
fontsize=10,
|
||||
bold=True,
|
||||
radius=0.10,
|
||||
)
|
||||
|
||||
# Output
|
||||
out_x = avg_x + avg_size / 2 + 0.50
|
||||
_arrow(ax, avg_x + avg_size / 2, avg_y, out_x, avg_y, lw=1.5, style="-|>")
|
||||
_draw_output(ax, out_x, avg_y)
|
||||
|
||||
# Side brackets — white text on badge
|
||||
_bracket(
|
||||
ax,
|
||||
x=br_od_x + 0.10,
|
||||
y0=od_y_md - 0.45,
|
||||
y1=od_y_img + 0.45,
|
||||
text="OD HyperTower",
|
||||
fontsize=8.5,
|
||||
pad=0.20,
|
||||
color="#555",
|
||||
badge_color="#555",
|
||||
)
|
||||
_bracket(
|
||||
ax,
|
||||
x=br_os_x + 0.10,
|
||||
y0=os_y_md - 0.45,
|
||||
y1=os_y_img + 0.45,
|
||||
text="OS HyperTower",
|
||||
fontsize=8.5,
|
||||
pad=0.20,
|
||||
color="#555",
|
||||
badge_color="#555",
|
||||
)
|
||||
|
||||
path = out_dir / "architecture_ensemble.png"
|
||||
fig.savefig(path, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {path}")
|
||||
|
||||
|
||||
# ── Figure 4: Fused Head ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def make_fused_head(out_dir: Path):
|
||||
W, H = 10.5, 7.5
|
||||
fig, ax = plt.subplots(figsize=(W, H))
|
||||
_setup(fig, ax, W, H, "Fused Head Bilateral HyperTower")
|
||||
|
||||
x_left = 3.0
|
||||
inp_cx = 1.85
|
||||
inp_w = 0.90
|
||||
inp_h = 0.55
|
||||
|
||||
# OD (top) — same layout as ensemble
|
||||
od_y_img, od_y_md = 5.90, 4.60
|
||||
br_od_x, cy_od = _draw_compact_ht(
|
||||
ax, x_left=x_left, y_img=od_y_img, y_md=od_y_md, eye_label="OD"
|
||||
)
|
||||
_text(ax, 0.45, (od_y_img + od_y_md) / 2, "OD\n(Right Eye)",
|
||||
fontsize=9, color="#444", bold=True)
|
||||
_box(ax, inp_cx, od_y_img, inp_w, inp_h, C_INPUT, "Fundus\nImage",
|
||||
fontsize=8, radius=0.08, alpha=0.75, text_color="#333")
|
||||
_box(ax, inp_cx, od_y_md, inp_w, inp_h, C_INPUT, "Clinical\nData",
|
||||
fontsize=8, radius=0.08, alpha=0.75, text_color="#333")
|
||||
_arrow(ax, inp_cx + inp_w / 2, od_y_img, x_left, od_y_img, lw=1.2, style="-|>")
|
||||
_arrow(ax, inp_cx + inp_w / 2, od_y_md, x_left, od_y_md, lw=1.2, style="-|>")
|
||||
|
||||
# OS (bottom)
|
||||
os_y_img, os_y_md = 2.80, 1.50
|
||||
br_os_x, cy_os = _draw_compact_ht(
|
||||
ax, x_left=x_left, y_img=os_y_img, y_md=os_y_md, eye_label="OS"
|
||||
)
|
||||
_text(ax, 0.45, (os_y_img + os_y_md) / 2, "OS\n(Left Eye)",
|
||||
fontsize=9, color="#444", bold=True)
|
||||
_box(ax, inp_cx, os_y_img, inp_w, inp_h, C_INPUT, "Fundus\nImage",
|
||||
fontsize=8, radius=0.08, alpha=0.75, text_color="#333")
|
||||
_box(ax, inp_cx, os_y_md, inp_w, inp_h, C_INPUT, "Clinical\nData",
|
||||
fontsize=8, radius=0.08, alpha=0.75, text_color="#333")
|
||||
_arrow(ax, inp_cx + inp_w / 2, os_y_img, x_left, os_y_img, lw=1.2, style="-|>")
|
||||
_arrow(ax, inp_cx + inp_w / 2, os_y_md, x_left, os_y_md, lw=1.2, style="-|>")
|
||||
|
||||
_bracket(
|
||||
ax,
|
||||
x=br_od_x + -0.17,
|
||||
y0=od_y_md - 0.45,
|
||||
y1=od_y_img + 0.45,
|
||||
text="OD HyperTower",
|
||||
fontsize=8.5,
|
||||
pad=0.20,
|
||||
color="#555",
|
||||
badge_color="#555",
|
||||
)
|
||||
_bracket(
|
||||
ax,
|
||||
x=br_os_x + -0.17,
|
||||
y0=os_y_md - 0.45,
|
||||
y1=os_y_img + 0.45,
|
||||
text="OS HyperTower",
|
||||
fontsize=8.5,
|
||||
pad=0.20,
|
||||
color="#555",
|
||||
badge_color="#555",
|
||||
)
|
||||
|
||||
# Fused Head box with logit MLP detail
|
||||
avg_y = (cy_od + cy_os) / 2
|
||||
head_x = max(br_od_x, br_os_x) + 2.20
|
||||
head_w = 1.80
|
||||
head_h = 1.20
|
||||
|
||||
_arrow(ax, br_od_x + 0.05, cy_od, head_x - head_w / 2, avg_y, lw=1.4, style="-|>")
|
||||
_arrow(ax, br_os_x + 0.05, cy_os, head_x - head_w / 2, avg_y, lw=1.4, style="-|>")
|
||||
_box(
|
||||
ax,
|
||||
head_x,
|
||||
avg_y,
|
||||
head_w,
|
||||
head_h,
|
||||
C_HEAD,
|
||||
"Fused Head\ncat(l_OD, l_OS)\n→ FC(64) → logits",
|
||||
fontsize=8.5,
|
||||
bold=False,
|
||||
radius=0.10,
|
||||
)
|
||||
|
||||
# Output nodes + softmax
|
||||
out_x = head_x + head_w / 2 + 0.50
|
||||
_arrow(ax, head_x + head_w / 2, avg_y, out_x, avg_y, lw=1.5, style="-|>")
|
||||
_draw_output(ax, out_x, avg_y)
|
||||
|
||||
path = out_dir / "architecture_fused_head.png"
|
||||
fig.savefig(path, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {path}")
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(
|
||||
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
||||
)
|
||||
ap.add_argument(
|
||||
"--out",
|
||||
type=Path,
|
||||
default=Path(__file__).resolve().parents[3] / "v3" / "figures",
|
||||
help="Output directory",
|
||||
)
|
||||
args = ap.parse_args()
|
||||
args.out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print("Generating architecture diagrams...")
|
||||
make_single_tower(args.out)
|
||||
make_hypertower(args.out)
|
||||
make_ensemble(args.out)
|
||||
make_fused_head(args.out)
|
||||
print("Done.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
Bar plot comparing CNN standalone vs HyperTower image_only AUC per backbone,
|
||||
with PAPILA paper reference lines.
|
||||
|
||||
Usage:
|
||||
python -m v3.scripts.output_analysis.plot_cnn_backbone_comparison \
|
||||
--results-dir v3/results/phase1 \
|
||||
--output v3/results/phase1/cnn_backbone_comparison.png
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.metrics import roc_auc_score
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
|
||||
|
||||
|
||||
BACKBONES = ["densenet121", "vgg16", "mobilenet_v2", "inception_v3", "resnet50"]
|
||||
BACKBONE_LABELS = {
|
||||
"densenet121": "DenseNet121",
|
||||
"vgg16": "VGG16",
|
||||
"mobilenet_v2": "MobileNetV2",
|
||||
"inception_v3": "Inception V3",
|
||||
"resnet50": "ResNet50",
|
||||
}
|
||||
|
||||
# Per-backbone paper AUCs (binary, Test #2, PAPILA 2022)
|
||||
PAPER_AUC = {
|
||||
"densenet121": 0.80,
|
||||
"vgg16": 0.84,
|
||||
"mobilenet_v2": 0.75,
|
||||
"inception_v3": 0.78,
|
||||
"resnet50": 0.78,
|
||||
}
|
||||
PAPER_STD = {
|
||||
"densenet121": 0.05,
|
||||
"vgg16": 0.02,
|
||||
"mobilenet_v2": 0.06,
|
||||
"inception_v3": 0.08,
|
||||
"resnet50": 0.07,
|
||||
}
|
||||
|
||||
COLOURS = {
|
||||
"cnn": "#4878CF",
|
||||
"ht": "#D65F5F",
|
||||
"paper_ref": "black",
|
||||
}
|
||||
|
||||
|
||||
def load_cnn_fold_aucs(results_dir: Path, backbone: str) -> list[float]:
|
||||
fpath = results_dir / f"cnn_{backbone}" / "fold_metrics.csv"
|
||||
if not fpath.exists():
|
||||
print(f" WARNING: missing {fpath}")
|
||||
return []
|
||||
df = pd.read_csv(fpath)
|
||||
return df["auc"].tolist()
|
||||
|
||||
|
||||
def load_ht_fold_aucs(results_dir: Path, backbone: str, n_folds: int = 5) -> list[float]:
|
||||
aucs = []
|
||||
for fold in range(n_folds):
|
||||
fold_dir = results_dir / "imageonly_ht" / backbone / "binary" / "single" / f"fold{fold}"
|
||||
y_path = fold_dir / "test_y_true.npy"
|
||||
p_path = fold_dir / "test_probs_fused.npy"
|
||||
if not (y_path.exists() and p_path.exists()):
|
||||
print(f" WARNING: missing predictions for {backbone} fold{fold}")
|
||||
continue
|
||||
y = np.load(y_path)
|
||||
pr = np.load(p_path)
|
||||
if len(np.unique(y)) < 2:
|
||||
print(f" WARNING: single-class test set for {backbone} fold{fold}, skipping")
|
||||
continue
|
||||
aucs.append(float(roc_auc_score(y, pr[:, 1])))
|
||||
return aucs
|
||||
|
||||
|
||||
def plot(cnn_data: dict, ht_data: dict, output: Path):
|
||||
n = len(BACKBONES)
|
||||
x = np.arange(n)
|
||||
group_width = 0.7
|
||||
bar_w = group_width / 2 * 0.88
|
||||
offsets = [-group_width / 4, group_width / 4]
|
||||
|
||||
fig, ax = plt.subplots(figsize=(10, 5.5))
|
||||
|
||||
for bi, backbone in enumerate(BACKBONES):
|
||||
for si, (tag, data, colour) in enumerate([
|
||||
("CNN standalone", cnn_data, COLOURS["cnn"]),
|
||||
("HyperTower (image only)", ht_data, COLOURS["ht"]),
|
||||
]):
|
||||
aucs = data.get(backbone, [])
|
||||
if not aucs:
|
||||
continue
|
||||
xpos = bi + offsets[si]
|
||||
mean, std = np.mean(aucs), np.std(aucs)
|
||||
ax.bar(
|
||||
xpos, mean, width=bar_w,
|
||||
color=colour, alpha=0.80,
|
||||
label=tag if bi == 0 else "_nolegend_",
|
||||
)
|
||||
ax.errorbar(
|
||||
xpos, mean, yerr=std,
|
||||
fmt="none", color="black", capsize=4, linewidth=1.2,
|
||||
)
|
||||
|
||||
# Paper reference line spanning this backbone's group
|
||||
paper_val = PAPER_AUC.get(backbone)
|
||||
if paper_val is not None:
|
||||
lw = group_width / 2 + bar_w / 2
|
||||
label = "PAPILA paper" if bi == 0 else "_nolegend_"
|
||||
ax.hlines(
|
||||
paper_val,
|
||||
bi - group_width / 2, bi + group_width / 2,
|
||||
colors=COLOURS["paper_ref"], linestyles=":", linewidths=1.8,
|
||||
label=label,
|
||||
)
|
||||
|
||||
ax.set_xticks(x)
|
||||
ax.set_xticklabels([BACKBONE_LABELS[b] for b in BACKBONES], fontsize=11)
|
||||
ax.set_ylabel("AUC (ROC)", fontsize=11)
|
||||
ax.set_title("Phase 1: CNN backbone AUC — standalone vs HyperTower (image only)", fontsize=12)
|
||||
ax.set_ylim(0.45, 1.02)
|
||||
ax.axhline(0.5, color="grey", linestyle="--", linewidth=0.8, alpha=0.4)
|
||||
ax.grid(axis="y", alpha=0.3, linestyle="--")
|
||||
ax.legend(loc="lower right", fontsize=10, framealpha=0.9)
|
||||
fig.tight_layout()
|
||||
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(output, dpi=180)
|
||||
plt.close(fig)
|
||||
print(f"Saved: {output}")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--results-dir", default="v3/results/phase1")
|
||||
ap.add_argument("--output", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
results_dir = Path(args.results_dir)
|
||||
output = Path(args.output) if args.output else results_dir / "cnn_backbone_comparison.png"
|
||||
|
||||
cnn_data = {b: load_cnn_fold_aucs(results_dir, b) for b in BACKBONES}
|
||||
ht_data = {b: load_ht_fold_aucs(results_dir, b) for b in BACKBONES}
|
||||
|
||||
plot(cnn_data, ht_data, output)
|
||||
|
||||
# Summary table
|
||||
print(f"\n{'Backbone':<16} {'CNN standalone':>18} {'HT image_only':>18} {'Paper':>12}")
|
||||
print("-" * 72)
|
||||
for b in BACKBONES:
|
||||
cnn_aucs = cnn_data[b]
|
||||
ht_aucs = ht_data[b]
|
||||
cnn_str = f"{np.mean(cnn_aucs):.3f} ± {np.std(cnn_aucs):.3f}" if cnn_aucs else "—"
|
||||
ht_str = f"{np.mean(ht_aucs):.3f} ± {np.std(ht_aucs):.3f}" if ht_aucs else "—"
|
||||
p_str = f"{PAPER_AUC[b]:.2f} ± {PAPER_STD[b]:.2f}"
|
||||
print(f"{BACKBONE_LABELS[b]:<16} {cnn_str:>18} {ht_str:>18} {p_str:>12}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,119 @@
|
||||
"""
|
||||
Phase 3 modality ablation — Image-only vs Clinical-only vs HyperTower (fused).
|
||||
|
||||
Pools all rep×fold predictions from the phase3/baseline run and plots
|
||||
per-fold AUC for each modality as a box plot with jittered points.
|
||||
|
||||
Output: v3/figures/phase3_modality_ablation.png
|
||||
|
||||
Usage:
|
||||
python -m v3.scripts.output_analysis.plot_phase3_modality_ablation
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.metrics import roc_auc_score
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
RESULTS_DIR = REPO_ROOT / "v3" / "results" / "phase3" / "baseline"
|
||||
FIGURES_DIR = REPO_ROOT / "v3" / "figures"
|
||||
OUT_PNG = FIGURES_DIR / "phase3_modality_ablation.png"
|
||||
|
||||
C_BASELINE = "#dd8452"
|
||||
C_OTHER = "#4c72b0"
|
||||
C_MEDIAN = "#c44e52"
|
||||
FSIZE = 10
|
||||
|
||||
MODALITIES = [
|
||||
("prob_img_c1", "Image only", C_OTHER),
|
||||
("prob_md_c1", "Clinical only", C_OTHER),
|
||||
("prob_fused_c1", "HyperTower\n(fused)", C_BASELINE),
|
||||
]
|
||||
|
||||
|
||||
def load_fold_aucs() -> dict[str, list[float]]:
|
||||
aucs: dict[str, list[float]] = {col: [] for col, _, _ in MODALITIES}
|
||||
|
||||
for rep_dir in sorted(RESULTS_DIR.glob("rep*")):
|
||||
fold_root = rep_dir / "binary" / "single"
|
||||
if not fold_root.exists():
|
||||
continue
|
||||
for fold_dir in sorted(fold_root.glob("fold[0-9]")):
|
||||
csv = fold_dir / "predictions_test.csv"
|
||||
if not csv.exists():
|
||||
continue
|
||||
df = pd.read_csv(csv)
|
||||
if df["y_true"].nunique() < 2:
|
||||
continue
|
||||
for col, _, _ in MODALITIES:
|
||||
if col in df.columns:
|
||||
try:
|
||||
aucs[col].append(roc_auc_score(df["y_true"], df[col]))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return aucs
|
||||
|
||||
|
||||
def main():
|
||||
print("Loading fold AUCs ...")
|
||||
aucs = load_fold_aucs()
|
||||
|
||||
n_folds = len(next(iter(aucs.values())))
|
||||
print(f" {n_folds} folds found")
|
||||
for col, label, _ in MODALITIES:
|
||||
vals = aucs[col]
|
||||
print(f" {label.replace(chr(10), ' '):<30} "
|
||||
f"mean={np.mean(vals):.4f} std={np.std(vals):.4f} n={len(vals)}")
|
||||
|
||||
# ── Plot ──────────────────────────────────────────────────────────────────
|
||||
fig, ax = plt.subplots(figsize=(6, 4.5))
|
||||
|
||||
data_list = [np.array(aucs[col]) for col, _, _ in MODALITIES]
|
||||
colors = [color for _, _, color in MODALITIES]
|
||||
labels = [lbl for _, lbl, _ in MODALITIES]
|
||||
x = np.arange(len(MODALITIES))
|
||||
|
||||
bp = ax.boxplot(
|
||||
data_list,
|
||||
vert=True,
|
||||
patch_artist=True,
|
||||
positions=x,
|
||||
widths=0.3,
|
||||
showfliers=True,
|
||||
flierprops=dict(marker="o", markersize=3, alpha=0.5),
|
||||
medianprops=dict(color=C_MEDIAN, linewidth=2),
|
||||
)
|
||||
for patch, color in zip(bp["boxes"], colors):
|
||||
patch.set_facecolor(color)
|
||||
patch.set_alpha(0.8)
|
||||
|
||||
ax.set_xlim(-0.5, len(MODALITIES) - 0.5)
|
||||
tick_labels = [
|
||||
f"{lbl}\nAUC={np.mean(np.array(aucs[col])):.3f}"
|
||||
for col, lbl, _ in MODALITIES
|
||||
]
|
||||
ax.set_xticks(x)
|
||||
ax.set_xticklabels(tick_labels, fontsize=FSIZE)
|
||||
ax.set_ylabel("AUC (ROC)", fontsize=FSIZE + 1)
|
||||
fig.suptitle(
|
||||
f"Phase 3 — Modality Ablation: Image / Clinical / Fused ({n_folds} folds)",
|
||||
fontsize=FSIZE + 3, fontweight="bold",
|
||||
)
|
||||
ax.grid(axis="y", alpha=0.3)
|
||||
|
||||
fig.tight_layout()
|
||||
FIGURES_DIR.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(OUT_PNG, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {OUT_PNG}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user