pre-refactor 041426
This commit is contained in:
rpotter6298
@@ -0,0 +1,264 @@
|
||||
"""
|
||||
Phase 5 comparison panel — ROC curves + fusion event summaries.
|
||||
|
||||
Layout:
|
||||
Top row (1 × 3) — ROC curves: Single HyperTower | Bilateral Ensemble | Fused Head
|
||||
Bottom rows (2 × 1) — Fusion event summary (full-width) for Ensemble then Fused Head
|
||||
(Single mode has fused-only bridge; no meaningful fusion events)
|
||||
|
||||
All data derived from predictions_test.csv — no checkpoints required.
|
||||
|
||||
Usage:
|
||||
python -m v3.scripts.output_analysis.explainability.comparison_panel_phase5
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.patches as mpatches
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.metrics import roc_auc_score, roc_curve
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
RESULTS_ROOT = REPO_ROOT / "v3" / "results"
|
||||
FIGURES_ROOT = REPO_ROOT / "v3" / "figures" / "explainability"
|
||||
CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
|
||||
|
||||
RUNS = [
|
||||
{"label": "Single HyperTower", "run": "phase5/single_fused",
|
||||
"tower_path": "binary/single", "is_single": False},
|
||||
{"label": "Bilateral Ensemble", "run": "phase5/ensemble_fused",
|
||||
"tower_path": "binary/ensemble", "is_single": False},
|
||||
{"label": "Fused Head", "run": "phase5/logit_mlp_head",
|
||||
"tower_path": "binary/ensemble", "is_single": False},
|
||||
]
|
||||
|
||||
# Event taxonomy (img=image tower, md=clinical tower)
|
||||
_EVENT_KEYS = [
|
||||
"full_correction", "img_assist", "md_assist",
|
||||
"full_error", "img_drag", "md_drag",
|
||||
"concordant_correct", "concordant_wrong",
|
||||
]
|
||||
_EVENT_COLORS = [
|
||||
"#2ca02c", "#98df8a", "#b5cf6b", # positive
|
||||
"#d62728", "#ff9896", "#ffbb78", # negative
|
||||
"#aec7e8", "#c5b0d5", # concordant
|
||||
]
|
||||
_POSITIVE_KEYS = _EVENT_KEYS[:3]
|
||||
_NEGATIVE_KEYS = _EVENT_KEYS[3:6]
|
||||
_DISAGREE_KEYS = _POSITIVE_KEYS + _NEGATIVE_KEYS # exclude concordant
|
||||
|
||||
|
||||
# ── Data loading ──────────────────────────────────────────────────────────────
|
||||
|
||||
def load_pooled(run: str, tower_path: str) -> pd.DataFrame:
|
||||
run_dir = RESULTS_ROOT / run
|
||||
rows = []
|
||||
for rep in sorted(run_dir.glob("rep*")):
|
||||
tm = rep / tower_path
|
||||
if not tm.exists():
|
||||
continue
|
||||
for fold in sorted(tm.glob("fold[0-9]")):
|
||||
csv = fold / "predictions_test.csv"
|
||||
if csv.exists():
|
||||
df = pd.read_csv(csv)
|
||||
df["rep"] = rep.name
|
||||
df["fold"] = fold.name
|
||||
rows.append(df)
|
||||
if not rows:
|
||||
raise FileNotFoundError(f"No predictions found under {run_dir}/{tower_path}")
|
||||
return pd.concat(rows, ignore_index=True)
|
||||
|
||||
|
||||
def classify_events(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Add event_type column based on pred_fused/pred_img/pred_md vs y_true."""
|
||||
df = df.copy()
|
||||
y = df["y_true"].values
|
||||
pf = df["pred_fused"].values
|
||||
pi = df["pred_img"].values
|
||||
pm = df["pred_md"].values
|
||||
|
||||
fused_ok = pf == y
|
||||
img_ok = pi == y
|
||||
md_ok = pm == y
|
||||
|
||||
def _classify(fo, io, mo):
|
||||
if fo and io and mo: return "concordant_correct"
|
||||
if not fo and not io and not mo: return "concordant_wrong"
|
||||
if fo and not io and not mo: return "full_correction"
|
||||
if fo and io and not mo: return "img_assist"
|
||||
if fo and not io and mo: return "md_assist"
|
||||
if not fo and io and mo: return "full_error"
|
||||
if not fo and not io and mo: return "img_drag"
|
||||
if not fo and io and not mo: return "md_drag"
|
||||
return "other"
|
||||
|
||||
df["event_type"] = [_classify(fo, io, mo)
|
||||
for fo, io, mo in zip(fused_ok, img_ok, md_ok)]
|
||||
# conf_delta: fused prob minus average of img/md
|
||||
df["conf_fused"] = df["prob_fused_c1"]
|
||||
df["conf_img"] = df["prob_img_c1"]
|
||||
df["conf_md"] = df["prob_md_c1"]
|
||||
df["conf_delta"] = df["conf_fused"] - 0.5 * (df["conf_img"] + df["conf_md"])
|
||||
return df
|
||||
|
||||
|
||||
# ── ROC panel ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def _draw_roc(ax, df: pd.DataFrame, label: str, color: str) -> None:
|
||||
"""Draw per-fold ROC curves (faint) + mean ROC (bold) on ax."""
|
||||
fold_aucs = []
|
||||
for (rep, fold), grp in df.groupby(["rep", "fold"]):
|
||||
if grp["y_true"].nunique() < 2:
|
||||
continue
|
||||
fpr, tpr, _ = roc_curve(grp["y_true"], grp["prob_fused_c1"])
|
||||
ax.plot(fpr, tpr, color=color, alpha=0.12, lw=0.8)
|
||||
fold_aucs.append(roc_auc_score(grp["y_true"], grp["prob_fused_c1"]))
|
||||
|
||||
# Mean ROC via interpolation
|
||||
mean_fpr = np.linspace(0, 1, 200)
|
||||
tprs = []
|
||||
for (rep, fold), grp in df.groupby(["rep", "fold"]):
|
||||
if grp["y_true"].nunique() < 2:
|
||||
continue
|
||||
fpr, tpr, _ = roc_curve(grp["y_true"], grp["prob_fused_c1"])
|
||||
tprs.append(np.interp(mean_fpr, fpr, tpr))
|
||||
mean_tpr = np.mean(tprs, axis=0)
|
||||
mean_auc = np.mean(fold_aucs)
|
||||
std_auc = np.std(fold_aucs)
|
||||
ax.plot(mean_fpr, mean_tpr, color=color, lw=2.2,
|
||||
label=f"Mean AUC = {mean_auc:.3f} ± {std_auc:.3f}")
|
||||
ax.fill_between(mean_fpr,
|
||||
np.percentile(tprs, 25, axis=0),
|
||||
np.percentile(tprs, 75, axis=0),
|
||||
color=color, alpha=0.12)
|
||||
ax.plot([0, 1], [0, 1], "k--", lw=0.7, alpha=0.5)
|
||||
ax.set_xlim(-0.02, 1.02); ax.set_ylim(-0.02, 1.02)
|
||||
ax.set_xlabel("False Positive Rate", fontsize=9)
|
||||
ax.set_ylabel("True Positive Rate", fontsize=9)
|
||||
ax.set_title(label, fontsize=10, fontweight="bold")
|
||||
ax.legend(fontsize=8, loc="lower right")
|
||||
ax.grid(alpha=0.25)
|
||||
|
||||
|
||||
# ── Fusion summary panel ──────────────────────────────────────────────────────
|
||||
|
||||
def _draw_fusion_summary(axes_row, df: pd.DataFrame, label: str) -> None:
|
||||
"""Draw 3-panel fusion summary (disagreement events only) on axes_row (list of 3 axes)."""
|
||||
event_color = dict(zip(_EVENT_KEYS, _EVENT_COLORS))
|
||||
event_labels = {
|
||||
"full_correction": "Full correction\n(both wrong → right)",
|
||||
"img_assist": "Img assist\n(img✓ md✗ → right)",
|
||||
"md_assist": "MD assist\n(md✓ img✗ → right)",
|
||||
"full_error": "Full error\n(both right → wrong)",
|
||||
"img_drag": "Img drag\n(img✗ md✓ → wrong)",
|
||||
"md_drag": "MD drag\n(md✗ img✓ → wrong)",
|
||||
}
|
||||
|
||||
# Only count disagreement events (exclude concordant)
|
||||
counts = {k: (df["event_type"] == k).sum() for k in _DISAGREE_KEYS}
|
||||
|
||||
# Panel 0: totals bar (positive vs negative)
|
||||
ax = axes_row[0]
|
||||
for bar_x, keys in ((0, _POSITIVE_KEYS), (1, _NEGATIVE_KEYS)):
|
||||
bot = 0
|
||||
for k in keys:
|
||||
c = int(counts[k])
|
||||
ax.bar(bar_x, c, bottom=bot, color=event_color[k], width=0.5)
|
||||
if c > 0:
|
||||
ax.text(bar_x, bot + c / 2, str(c), ha="center", va="center",
|
||||
fontsize=8, fontweight="bold")
|
||||
bot += c
|
||||
ax.set_xticks([0, 1]); ax.set_xticklabels(["Positive\nevents", "Negative\nevents"])
|
||||
ax.set_ylabel("Count (all folds)")
|
||||
patches = [mpatches.Patch(color=event_color[k], label=event_labels[k].split("\n")[0])
|
||||
for k in _DISAGREE_KEYS if counts[k] > 0]
|
||||
ax.legend(handles=patches, fontsize=6, loc="upper right")
|
||||
ax.set_title(f"{label}\nDisagreement event totals", fontsize=9)
|
||||
|
||||
# Panel 1: per-fold stacked bar (disagreement events only)
|
||||
ax = axes_row[1]
|
||||
fold_groups = sorted(df.groupby(["rep", "fold"]), key=lambda x: x[0])
|
||||
x = np.arange(len(fold_groups))
|
||||
pos_bot = np.zeros(len(fold_groups))
|
||||
neg_bot = np.zeros(len(fold_groups))
|
||||
for k, color in zip(_POSITIVE_KEYS, _EVENT_COLORS[:3]):
|
||||
vals = np.array([(g["event_type"] == k).sum() for _, g in fold_groups], dtype=float)
|
||||
ax.bar(x, vals, bottom=pos_bot, color=color, width=0.6)
|
||||
pos_bot += vals
|
||||
for k, color in zip(_NEGATIVE_KEYS, _EVENT_COLORS[3:6]):
|
||||
vals = np.array([(g["event_type"] == k).sum() for _, g in fold_groups], dtype=float)
|
||||
ax.bar(x + 0.65, vals, bottom=neg_bot, color=color, width=0.6)
|
||||
neg_bot += vals
|
||||
ax.set_xticks([])
|
||||
ax.set_xlabel("Fold", fontsize=8)
|
||||
ax.set_ylabel("Count"); ax.set_title("Per-fold breakdown\n(left=positive, right=negative)", fontsize=9)
|
||||
|
||||
# Panel 2: img vs md confidence scatter (disagreement events only)
|
||||
ax = axes_row[2]
|
||||
for k in _DISAGREE_KEYS:
|
||||
sub = df[df["event_type"] == k]
|
||||
if len(sub) == 0:
|
||||
continue
|
||||
ax.scatter(sub["conf_img"], sub["conf_md"], c=event_color[k],
|
||||
alpha=0.65, s=30, edgecolors="none",
|
||||
label=event_labels[k].split("\n")[0])
|
||||
ax.plot([0, 1], [0, 1], "k--", lw=0.5, alpha=0.4)
|
||||
ax.set_xlabel("P(Glaucoma) — Image head"); ax.set_ylabel("P(Glaucoma) — MD head")
|
||||
ax.legend(fontsize=5.5, loc="lower right")
|
||||
ax.set_title("Tower confidence space\n(disagreement events only)", fontsize=9)
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
ROC_COLORS = ["#4c72b0", "#dd8452", "#55a868"]
|
||||
|
||||
print("Loading predictions ...")
|
||||
datasets = []
|
||||
for cfg, color in zip(RUNS, ROC_COLORS):
|
||||
df = load_pooled(cfg["run"], cfg["tower_path"])
|
||||
df = classify_events(df)
|
||||
datasets.append((cfg, df, color))
|
||||
|
||||
fusion_runs = [(cfg, df, color) for cfg, df, color in datasets]
|
||||
|
||||
# ── Layout ────────────────────────────────────────────────────────────────
|
||||
# Row 0: 3 ROC axes
|
||||
# Rows 1,2: 2 fusion summary strips (5 axes each, spanning full width)
|
||||
n_fusion = len(fusion_runs)
|
||||
fig = plt.figure(figsize=(20, 6 + 4.5 * n_fusion))
|
||||
gs = fig.add_gridspec(
|
||||
1 + n_fusion, 1,
|
||||
height_ratios=[5] + [4.5] * n_fusion,
|
||||
hspace=0.35,
|
||||
)
|
||||
|
||||
# ROC row — subdivide into 3
|
||||
roc_gs = gs[0].subgridspec(1, 3, wspace=0.28)
|
||||
for i, (cfg, df, color) in enumerate(datasets):
|
||||
ax = fig.add_subplot(roc_gs[i])
|
||||
_draw_roc(ax, df, cfg["label"], color)
|
||||
|
||||
# Fusion rows (3 panels each)
|
||||
for fi, (cfg, df, color) in enumerate(fusion_runs):
|
||||
fus_gs = gs[1 + fi].subgridspec(1, 3, wspace=0.32)
|
||||
axes_row = [fig.add_subplot(fus_gs[j]) for j in range(3)]
|
||||
_draw_fusion_summary(axes_row, df, cfg["label"])
|
||||
|
||||
fig.suptitle("Phase 5 — Model Comparison: ROC Curves & Fusion Event Analysis",
|
||||
fontsize=13, fontweight="bold", y=1.01)
|
||||
|
||||
out = FIGURES_ROOT / "comparison_panel_phase5.png"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,615 @@
|
||||
"""
|
||||
Phase 5 explainability — confidence strips and head comparison.
|
||||
|
||||
Works from saved prediction CSVs. If patient_id column is present (requires
|
||||
a re-run after the v3_hypertower.py update), points are colored by VFI
|
||||
severity group. Otherwise falls back to a single color per true class.
|
||||
|
||||
VFI severity groups (VF_MD from clinical data):
|
||||
Early VF_MD > -6
|
||||
Moderate VF_MD -6 to -12
|
||||
Severe VF_MD < -12
|
||||
|
||||
Produces (all in figures/explainability/):
|
||||
confidence_strips.png — vertical strip: P(glaucoma) by true class, VFI colored
|
||||
head_comparison.png — fused vs img vs md distributions side by side
|
||||
|
||||
Usage:
|
||||
python -m v3.scripts.output_analysis.explainability.confidence_strips
|
||||
python -m v3.scripts.output_analysis.explainability.confidence_strips \
|
||||
--run phase5/logit_mlp_head --clinical-dir Papila/ClinicalData
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.patches as mpatches
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
RESULTS_ROOT = REPO_ROOT / "v3" / "results"
|
||||
FIGURES_ROOT = REPO_ROOT / "v3" / "figures"
|
||||
CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
|
||||
|
||||
FONT = "DejaVu Sans"
|
||||
|
||||
# Colour palette
|
||||
C_NORMAL = "#78909C" # blue-grey — healthy controls (no VFI staging)
|
||||
C_EARLY = "#29B6F6" # sky-blue — glaucoma, early VFI loss
|
||||
C_MODERATE = "#FFB300" # amber — glaucoma, moderate VFI loss
|
||||
C_SEVERE = "#E53935" # vivid red — glaucoma, severe VFI loss
|
||||
C_UNKNOWN = "#BDBDBD" # light grey — glaucoma, VFI not recorded
|
||||
|
||||
SEV_LABELS = {
|
||||
"normal": "Normal",
|
||||
"early": "Glaucoma — early (VF_MD > −6)",
|
||||
"moderate": "Glaucoma — moderate (−12 to −6)",
|
||||
"severe": "Glaucoma — severe (VF_MD < −12)",
|
||||
"unknown": "Glaucoma — VF_MD not recorded",
|
||||
}
|
||||
SEV_COLORS = {
|
||||
"normal": C_NORMAL,
|
||||
"early": C_EARLY,
|
||||
"moderate": C_MODERATE,
|
||||
"severe": C_SEVERE,
|
||||
"unknown": C_UNKNOWN,
|
||||
}
|
||||
|
||||
HEAD_COLORS = {"fused": "#d4a017", "img": "#4e8d3a", "md": "#4c72b0"}
|
||||
HEAD_LABELS = {
|
||||
"fused": "Fused head",
|
||||
"img": "Image-only head",
|
||||
"md": "Clinical-only head",
|
||||
}
|
||||
|
||||
|
||||
# ── VFI data ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def load_vfi(clinical_dir: Path) -> pd.DataFrame:
|
||||
"""Return DataFrame with columns [patient_id (int), vf_md (float), severity (str)].
|
||||
Only includes patients in the binary study (Diagnosis 0=Normal, 1=Glaucoma).
|
||||
"""
|
||||
od = pd.read_excel(clinical_dir / "patient_data_od.xlsx", header=1)
|
||||
os_ = pd.read_excel(clinical_dir / "patient_data_os.xlsx", header=1)
|
||||
|
||||
def _clean(df, eye):
|
||||
df = df.copy()
|
||||
if "Patient ID" not in df.columns and "ID" in df.columns:
|
||||
df.rename(columns={"ID": "Patient ID"}, inplace=True)
|
||||
df["Patient ID"] = (
|
||||
df["Patient ID"].astype(str).str.extract(r"(\d+)")[0].astype(int)
|
||||
)
|
||||
df["Diagnosis"] = pd.to_numeric(df["Diagnosis"], errors="coerce")
|
||||
df["VF_MD"] = pd.to_numeric(df["VF_MD"], errors="coerce")
|
||||
# PAPILA: 0=Normal, 1=Glaucoma, 2=Suspect — keep only binary patients
|
||||
df = df[df["Diagnosis"].isin([0, 1])].copy()
|
||||
df["eye"] = eye
|
||||
return df[["Patient ID", "Diagnosis", "VF_MD", "eye"]]
|
||||
|
||||
combined = pd.concat([_clean(od, "OD"), _clean(os_, "OS")], ignore_index=True)
|
||||
|
||||
# Per patient: modal diagnosis, worst (most negative) VF_MD across eyes
|
||||
diag = (
|
||||
combined.groupby("Patient ID")["Diagnosis"]
|
||||
.agg(lambda x: x.mode().iloc[0])
|
||||
.reset_index()
|
||||
)
|
||||
vf = combined.groupby("Patient ID")["VF_MD"].min().reset_index()
|
||||
worst = diag.merge(vf, on="Patient ID").rename(
|
||||
columns={"Patient ID": "patient_id", "VF_MD": "vf_md", "Diagnosis": "diagnosis"}
|
||||
)
|
||||
|
||||
def _severity(row):
|
||||
if int(row["diagnosis"]) == 0:
|
||||
return "normal" # healthy control — no VFI staging
|
||||
v = row["vf_md"]
|
||||
if pd.isna(v):
|
||||
return "unknown" # glaucoma, no VFI recorded
|
||||
if v > -6:
|
||||
return "early"
|
||||
if v > -12:
|
||||
return "moderate"
|
||||
return "severe"
|
||||
|
||||
worst["severity"] = worst.apply(_severity, axis=1)
|
||||
return worst
|
||||
|
||||
|
||||
# ── Prediction loading ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def load_all_predictions(
|
||||
run_dir: Path, tower_path: str = "binary/ensemble"
|
||||
) -> pd.DataFrame:
|
||||
"""Pool predictions_test.csv across all reps and folds."""
|
||||
rows = []
|
||||
for rep_dir in sorted(run_dir.glob("rep*")):
|
||||
tm_dir = rep_dir / tower_path
|
||||
if not tm_dir.exists():
|
||||
continue
|
||||
for fold_dir in sorted(tm_dir.glob("fold[0-9]")):
|
||||
csv_path = fold_dir / "predictions_test.csv"
|
||||
if not csv_path.exists():
|
||||
continue
|
||||
df = pd.read_csv(csv_path)
|
||||
df["rep"] = rep_dir.name
|
||||
df["fold"] = fold_dir.name
|
||||
rows.append(df)
|
||||
if not rows:
|
||||
raise FileNotFoundError(f"No predictions_test.csv found under {run_dir}")
|
||||
return pd.concat(rows, ignore_index=True)
|
||||
|
||||
|
||||
# ── Figure 1: Confidence strips (vertical) ───────────────────────────────────
|
||||
|
||||
|
||||
def make_confidence_strips(
|
||||
df: pd.DataFrame, vfi: pd.DataFrame | None, out_path: Path
|
||||
) -> None:
|
||||
"""
|
||||
Vertical strip plot: x = true class, y = P(glaucoma).
|
||||
Points colored by VFI severity if patient_id column available, else uniform.
|
||||
"""
|
||||
rng = np.random.default_rng(42)
|
||||
has_vfi = (
|
||||
vfi is not None
|
||||
and "patient_id" in df.columns
|
||||
and df["patient_id"].notna().any()
|
||||
)
|
||||
|
||||
if has_vfi:
|
||||
df = df.copy()
|
||||
df["patient_id"] = pd.to_numeric(df["patient_id"], errors="coerce").astype(
|
||||
"Int64"
|
||||
)
|
||||
vfi_merge = vfi.copy()
|
||||
vfi_merge["patient_id"] = vfi_merge["patient_id"].astype("Int64")
|
||||
df = df.merge(
|
||||
vfi_merge[["patient_id", "severity"]], on="patient_id", how="left"
|
||||
)
|
||||
df["severity"] = df["severity"].fillna("unknown")
|
||||
else:
|
||||
df["severity"] = "unknown"
|
||||
|
||||
fig, ax = plt.subplots(figsize=(6, 7))
|
||||
fig.patch.set_facecolor("#e8e8e8")
|
||||
ax.set_facecolor("#e8e8e8")
|
||||
|
||||
x_pos = {0: 0.0, 1: 1.0}
|
||||
jitter_scale = 0.18
|
||||
|
||||
# Draw in severity order so severe is on top
|
||||
sev_order = ["normal", "unknown", "early", "moderate", "severe"]
|
||||
sev_alpha = {
|
||||
"normal": 0.40,
|
||||
"early": 0.55,
|
||||
"moderate": 0.70,
|
||||
"severe": 0.85,
|
||||
"unknown": 0.35,
|
||||
}
|
||||
sev_size = {"normal": 6, "early": 8, "moderate": 10, "severe": 12, "unknown": 6}
|
||||
|
||||
for sev in sev_order:
|
||||
mask = df["severity"] == sev
|
||||
if not mask.any():
|
||||
continue
|
||||
sub = df[mask]
|
||||
jitter = rng.uniform(-jitter_scale, jitter_scale, len(sub))
|
||||
x = np.array([x_pos[int(v)] for v in sub["y_true"]]) + jitter
|
||||
ax.scatter(
|
||||
x,
|
||||
sub["prob_fused_c1"].values,
|
||||
c=SEV_COLORS[sev],
|
||||
s=sev_size[sev],
|
||||
alpha=sev_alpha[sev],
|
||||
linewidths=0,
|
||||
zorder=3,
|
||||
label=SEV_LABELS[sev],
|
||||
)
|
||||
|
||||
# Median lines per class
|
||||
for cls, xc in x_pos.items():
|
||||
med = np.median(df.loc[df["y_true"] == cls, "prob_fused_c1"])
|
||||
ax.plot(
|
||||
[xc - jitter_scale - 0.04, xc + jitter_scale + 0.04],
|
||||
[med, med],
|
||||
color="#222",
|
||||
lw=2.0,
|
||||
zorder=5,
|
||||
)
|
||||
|
||||
ax.axhline(0.5, color="#888", lw=1.2, ls="--", alpha=0.7, zorder=2)
|
||||
ax.set_xticks([0, 1])
|
||||
ax.set_xticklabels(["Normal", "Glaucoma"], fontsize=11)
|
||||
ax.set_ylabel("Predicted P(Glaucoma)", fontsize=11)
|
||||
ax.set_ylim(-0.04, 1.04)
|
||||
ax.set_xlim(-0.55, 1.55)
|
||||
ax.set_title(
|
||||
"Confidence Strips — Fused Head\n(Phase 5, all folds)",
|
||||
fontsize=12,
|
||||
fontweight="bold",
|
||||
)
|
||||
ax.grid(axis="y", alpha=0.3, zorder=1)
|
||||
|
||||
# Legend — only show groups that appear
|
||||
handles, labels = ax.get_legend_handles_labels()
|
||||
if handles:
|
||||
ax.legend(
|
||||
handles=handles,
|
||||
labels=labels,
|
||||
fontsize=8.5,
|
||||
loc="upper center",
|
||||
framealpha=0.75,
|
||||
ncol=2,
|
||||
)
|
||||
|
||||
if not has_vfi:
|
||||
ax.text(
|
||||
0.98,
|
||||
0.02,
|
||||
"Re-run with updated v3_hypertower.py\nto enable VFI severity coloring",
|
||||
transform=ax.transAxes,
|
||||
ha="right",
|
||||
va="bottom",
|
||||
fontsize=7.5,
|
||||
color="#888",
|
||||
style="italic",
|
||||
)
|
||||
|
||||
fig.tight_layout()
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out_path, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {out_path}")
|
||||
|
||||
|
||||
# ── Figure 2: Head comparison ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def make_head_comparison(df: pd.DataFrame, out_path: Path) -> None:
|
||||
"""Side-by-side violin + strip of P(glaucoma) by true class for each head."""
|
||||
heads = ["fused", "img", "md"]
|
||||
prob_cols = {"fused": "prob_fused_c1", "img": "prob_img_c1", "md": "prob_md_c1"}
|
||||
|
||||
rng = np.random.default_rng(42)
|
||||
fig, axes = plt.subplots(1, 3, figsize=(11, 4.5), sharey=True)
|
||||
fig.patch.set_facecolor("#e8e8e8")
|
||||
fig.suptitle(
|
||||
"Head Comparison — P(Glaucoma) by True Class (Phase 5, all folds)",
|
||||
fontsize=12,
|
||||
fontweight="bold",
|
||||
)
|
||||
|
||||
c_normal = "#4c72b0"
|
||||
c_glaucoma = "#c44e52"
|
||||
|
||||
for ax, head in zip(axes, heads):
|
||||
ax.set_facecolor("#e8e8e8")
|
||||
col = prob_cols[head]
|
||||
data_by_class = [df.loc[df["y_true"] == cls, col].values for cls in [0, 1]]
|
||||
|
||||
vp = ax.violinplot(
|
||||
data_by_class,
|
||||
positions=[0, 1],
|
||||
widths=0.6,
|
||||
showmedians=True,
|
||||
showextrema=False,
|
||||
)
|
||||
for body, color in zip(vp["bodies"], [c_normal, c_glaucoma]):
|
||||
body.set_facecolor(color)
|
||||
body.set_alpha(0.35)
|
||||
vp["cmedians"].set_color("#222")
|
||||
vp["cmedians"].set_linewidth(2)
|
||||
|
||||
for cls, color in zip([0, 1], [c_normal, c_glaucoma]):
|
||||
vals = data_by_class[cls]
|
||||
jitter = rng.uniform(-0.12, 0.12, len(vals))
|
||||
ax.scatter(
|
||||
cls + jitter, vals, color=color, s=4, alpha=0.30, linewidths=0, zorder=3
|
||||
)
|
||||
|
||||
ax.axhline(0.5, color="#888", lw=1.0, ls="--", alpha=0.6)
|
||||
ax.set_xticks([0, 1])
|
||||
ax.set_xticklabels(["Normal", "Glaucoma"], fontsize=9)
|
||||
ax.set_title(
|
||||
HEAD_LABELS[head], fontsize=10, fontweight="bold", color=HEAD_COLORS[head]
|
||||
)
|
||||
ax.set_ylim(-0.05, 1.05)
|
||||
ax.grid(axis="y", alpha=0.3)
|
||||
if head == "fused":
|
||||
ax.set_ylabel("Predicted P(Glaucoma)", fontsize=10)
|
||||
|
||||
from sklearn.metrics import roc_auc_score
|
||||
|
||||
try:
|
||||
auc = roc_auc_score(df["y_true"], df[col])
|
||||
ax.text(
|
||||
0.97,
|
||||
0.04,
|
||||
f"AUC = {auc:.3f}",
|
||||
transform=ax.transAxes,
|
||||
ha="right",
|
||||
va="bottom",
|
||||
fontsize=9,
|
||||
color="#333",
|
||||
bbox=dict(facecolor="white", alpha=0.65, edgecolor="none", pad=2),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
fig.tight_layout()
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out_path, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {out_path}")
|
||||
|
||||
|
||||
# ── Multi-model comparison strip ──────────────────────────────────────────────
|
||||
|
||||
|
||||
def make_comparison_strips(
|
||||
run_configs: list[dict], vfi: pd.DataFrame, out_path: Path
|
||||
) -> None:
|
||||
"""
|
||||
Side-by-side confidence strips for multiple runs.
|
||||
Each config: {"label": str, "df": DataFrame}.
|
||||
"""
|
||||
from sklearn.metrics import roc_auc_score
|
||||
|
||||
rng = np.random.default_rng(42)
|
||||
|
||||
n = len(run_configs)
|
||||
fig, axes = plt.subplots(1, n, figsize=(4.5 * n, 7), sharey=True)
|
||||
if n == 1:
|
||||
axes = [axes]
|
||||
fig.patch.set_facecolor("#e8e8e8")
|
||||
fig.suptitle(
|
||||
"Confidence Strips by Model (Phase 5, all folds)",
|
||||
fontsize=13,
|
||||
fontweight="bold",
|
||||
)
|
||||
|
||||
sev_order = ["normal", "unknown", "early", "moderate", "severe"]
|
||||
sev_alpha = {
|
||||
"normal": 0.40,
|
||||
"early": 0.55,
|
||||
"moderate": 0.70,
|
||||
"severe": 0.85,
|
||||
"unknown": 0.35,
|
||||
}
|
||||
sev_size = {"normal": 6, "early": 8, "moderate": 10, "severe": 12, "unknown": 6}
|
||||
x_pos = {0: 0.0, 1: 1.0}
|
||||
jitter_scale = 0.18
|
||||
|
||||
for ax, cfg in zip(axes, run_configs):
|
||||
ax.set_facecolor("#e8e8e8")
|
||||
df = cfg["df"]
|
||||
|
||||
# Attach VFI severity
|
||||
df = df.copy()
|
||||
df["patient_id"] = pd.to_numeric(df["patient_id"], errors="coerce").astype(
|
||||
"Int64"
|
||||
)
|
||||
vfi_m = vfi.copy()
|
||||
vfi_m["patient_id"] = vfi_m["patient_id"].astype("Int64")
|
||||
df = df.merge(vfi_m[["patient_id", "severity"]], on="patient_id", how="left")
|
||||
df["severity"] = df["severity"].fillna("unknown")
|
||||
|
||||
for sev in sev_order:
|
||||
mask = df["severity"] == sev
|
||||
if not mask.any():
|
||||
continue
|
||||
sub = df[mask]
|
||||
jitter = rng.uniform(-jitter_scale, jitter_scale, len(sub))
|
||||
x = np.array([x_pos[int(v)] for v in sub["y_true"]]) + jitter
|
||||
ax.scatter(
|
||||
x,
|
||||
sub["prob_fused_c1"].values,
|
||||
c=SEV_COLORS[sev],
|
||||
s=sev_size[sev],
|
||||
alpha=sev_alpha[sev],
|
||||
linewidths=0,
|
||||
zorder=3,
|
||||
)
|
||||
|
||||
# IQR box + median line per class
|
||||
iqr_w = 0.06
|
||||
xticklabels = []
|
||||
for cls, xc in x_pos.items():
|
||||
vals = df.loc[df["y_true"] == cls, "prob_fused_c1"]
|
||||
med = np.median(vals)
|
||||
q25 = np.percentile(vals, 25)
|
||||
q75 = np.percentile(vals, 75)
|
||||
# Subtle translucent IQR box
|
||||
ax.add_patch(
|
||||
plt.Rectangle(
|
||||
(xc - iqr_w, q25),
|
||||
2 * iqr_w,
|
||||
q75 - q25,
|
||||
facecolor="#555",
|
||||
alpha=0.18,
|
||||
linewidth=0,
|
||||
zorder=4,
|
||||
)
|
||||
)
|
||||
# Median line
|
||||
ax.plot(
|
||||
[xc - jitter_scale - 0.04, xc + jitter_scale + 0.04],
|
||||
[med, med],
|
||||
color="#222",
|
||||
lw=2.0,
|
||||
zorder=5,
|
||||
label="Median" if cls == 0 else None,
|
||||
)
|
||||
# TP / TN rate below x-label
|
||||
if cls == 1:
|
||||
rate = (vals > 0.5).mean() * 100
|
||||
xticklabels.append(f"Glaucoma\nTP {rate:.0f}%")
|
||||
else:
|
||||
rate = (vals <= 0.5).mean() * 100
|
||||
xticklabels.append(f"Normal\nTN {rate:.0f}%")
|
||||
|
||||
ax.axhline(0.5, color="#888", lw=1.2, ls="--", alpha=0.7, zorder=2)
|
||||
ax.set_xticks([0, 1])
|
||||
ax.set_xticklabels(xticklabels, fontsize=10)
|
||||
ax.set_title(cfg["label"], fontsize=11, fontweight="bold")
|
||||
ax.set_ylim(-0.04, 1.04)
|
||||
ax.set_xlim(-0.55, 1.55)
|
||||
ax.grid(axis="y", alpha=0.3, zorder=1)
|
||||
|
||||
try:
|
||||
fold_aucs = [
|
||||
roc_auc_score(g["y_true"], g["prob_fused_c1"])
|
||||
for _, g in df.groupby(["rep", "fold"])
|
||||
if g["y_true"].nunique() > 1
|
||||
]
|
||||
mean_auc = np.mean(fold_aucs)
|
||||
std_auc = np.std(fold_aucs)
|
||||
ax.text(
|
||||
0.65,
|
||||
0.0,
|
||||
f"AUC = {mean_auc:.3f} ± {std_auc:.3f}",
|
||||
transform=ax.transAxes,
|
||||
ha="right",
|
||||
va="bottom",
|
||||
fontsize=9,
|
||||
color="#333",
|
||||
bbox=dict(facecolor="white", alpha=0.65, edgecolor="none", pad=2),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
axes[0].set_ylabel("Predicted P(Glaucoma)", fontsize=11)
|
||||
|
||||
# Shared legend
|
||||
legend_patches = [
|
||||
mpatches.Patch(color=SEV_COLORS[s], label=SEV_LABELS[s])
|
||||
for s in ["normal", "early", "moderate", "severe", "unknown"]
|
||||
]
|
||||
fig.legend(
|
||||
handles=legend_patches,
|
||||
fontsize=9,
|
||||
loc="lower center",
|
||||
ncol=len(legend_patches),
|
||||
framealpha=0.75,
|
||||
bbox_to_anchor=(0.5, -0.02),
|
||||
)
|
||||
|
||||
fig.tight_layout(rect=[0, 0.06, 1, 1])
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out_path, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f" Saved: {out_path}")
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Default runs shown in the comparison
|
||||
DEFAULT_RUNS = [
|
||||
{
|
||||
"run": "phase4/single",
|
||||
"tower_path": "binary/single",
|
||||
"label": "Single HyperTower",
|
||||
},
|
||||
{
|
||||
"run": "phase5/ensemble_fused",
|
||||
"tower_path": "binary/ensemble",
|
||||
"label": "Bilateral Ensemble",
|
||||
},
|
||||
{
|
||||
"run": "phase5/logit_mlp_head",
|
||||
"tower_path": "binary/ensemble",
|
||||
"label": "Fused Head",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _aggregate_eye_to_patient(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Single-mode predictions are eye-level (2 rows per patient per fold).
|
||||
In the test loader, OD rows come first (sorted patient ID order) then OS.
|
||||
Average the two eyes to get one patient-level row per fold.
|
||||
"""
|
||||
rows = []
|
||||
prob_cols = [c for c in df.columns if c.startswith("prob_")]
|
||||
pred_cols = [c for c in df.columns if c.startswith("pred_")]
|
||||
|
||||
for (rep, fold), grp in df.groupby(["rep", "fold"]):
|
||||
n = len(grp)
|
||||
half = n // 2
|
||||
od = grp.iloc[:half].reset_index(drop=True)
|
||||
os_ = grp.iloc[half:].reset_index(drop=True)
|
||||
pat = od.copy()
|
||||
for col in prob_cols:
|
||||
pat[col] = (od[col].values + os_[col].values) / 2
|
||||
for col in pred_cols:
|
||||
pat[col] = (pat[col.replace("pred_", "prob_") + "_c1"] >= 0.5).astype(int)
|
||||
rows.append(pat)
|
||||
|
||||
return pd.concat(rows, ignore_index=True)
|
||||
|
||||
|
||||
def _load_run(run: str, tower_path: str, clinical_dir: Path) -> pd.DataFrame:
|
||||
run_dir = RESULTS_ROOT / run
|
||||
print(f" Loading {run} ...")
|
||||
df = load_all_predictions(run_dir, tower_path=tower_path)
|
||||
if tower_path.endswith("/single"):
|
||||
from v3.scripts.output_analysis.explainability.fold_patient_ids import (
|
||||
attach_patient_ids_single,
|
||||
)
|
||||
|
||||
df = attach_patient_ids_single(df, clinical_dir=clinical_dir, batch_size=8)
|
||||
else:
|
||||
from v3.scripts.output_analysis.explainability.fold_patient_ids import (
|
||||
attach_patient_ids,
|
||||
)
|
||||
|
||||
df = attach_patient_ids(df, clinical_dir=clinical_dir)
|
||||
return df
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(
|
||||
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
||||
)
|
||||
ap.add_argument(
|
||||
"--run", default="phase5/logit_mlp_head", help="Single run for standalone plots"
|
||||
)
|
||||
ap.add_argument("--tower-path", default="binary/ensemble")
|
||||
ap.add_argument("--clinical-dir", type=Path, default=CLINICAL_DIR)
|
||||
ap.add_argument("--out", type=Path, default=FIGURES_ROOT / "explainability")
|
||||
args = ap.parse_args()
|
||||
|
||||
print("Loading VFI data ...")
|
||||
vfi = load_vfi(args.clinical_dir)
|
||||
|
||||
# ── Single-run plots (strips + head comparison) ──────────────────────────
|
||||
df = _load_run(args.run, args.tower_path, args.clinical_dir)
|
||||
make_confidence_strips(df, vfi, args.out / "confidence_strips.png")
|
||||
make_head_comparison(df, args.out / "head_comparison.png")
|
||||
|
||||
# ── Multi-model comparison ───────────────────────────────────────────────
|
||||
print("Building comparison strips ...")
|
||||
run_configs = []
|
||||
for cfg in DEFAULT_RUNS:
|
||||
try:
|
||||
df_r = _load_run(cfg["run"], cfg["tower_path"], args.clinical_dir)
|
||||
run_configs.append({"label": cfg["label"], "df": df_r})
|
||||
except FileNotFoundError as e:
|
||||
print(f" Skipping {cfg['run']}: {e}")
|
||||
if run_configs:
|
||||
make_comparison_strips(
|
||||
run_configs, vfi, args.out / "confidence_strips_comparison.png"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,181 @@
|
||||
"""
|
||||
Derives test-set patient IDs for any (rep, fold) without re-running training.
|
||||
|
||||
The split is fully deterministic: fold_seed = rep_seed_start + rep * rep_seed_step.
|
||||
build_samples() groups by patient_id with sort=True (pandas default), and the
|
||||
test DataLoader uses shuffle=False — so rows in predictions_test.csv are always
|
||||
in ascending Patient ID order within each test fold.
|
||||
|
||||
Usage:
|
||||
from v3.scripts.output_analysis.explainability.fold_patient_ids import get_test_patient_ids
|
||||
pids = get_test_patient_ids(rep=0, fold=2) # list of int patient IDs, sorted
|
||||
|
||||
# Attach to a pooled predictions DataFrame:
|
||||
df = attach_patient_ids(df, clinical_dir=...)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
|
||||
|
||||
# These match the defaults in run_cv.py
|
||||
_REP_SEED_START = 100
|
||||
_REP_SEED_STEP = 100
|
||||
_N_SPLITS = 5
|
||||
_EVAL_MODE = "binary"
|
||||
_LABEL_COL = "Diagnosis"
|
||||
_PATIENT_COL = "Patient ID"
|
||||
|
||||
|
||||
@lru_cache(maxsize=4)
|
||||
def _load_clinical(clinical_dir: Path) -> pd.DataFrame:
|
||||
"""
|
||||
Load OD/OS Excel sheets, extract Patient ID + Diagnosis, binary-filter.
|
||||
Returns a DataFrame with one row per eye (OD+OS stacked), columns:
|
||||
[Patient ID, Diagnosis, eyeID, VF_MD].
|
||||
"""
|
||||
od = pd.read_excel(clinical_dir / "patient_data_od.xlsx", header=1)
|
||||
os_ = pd.read_excel(clinical_dir / "patient_data_os.xlsx", header=1)
|
||||
od["eyeID"] = "OD"
|
||||
os_["eyeID"] = "OS"
|
||||
df = pd.concat([od, os_], ignore_index=True)
|
||||
# Raw column is "ID" (e.g. "#002"); canonicalize to "Patient ID"
|
||||
if "Patient ID" not in df.columns and "ID" in df.columns:
|
||||
df.rename(columns={"ID": "Patient ID"}, inplace=True)
|
||||
df["Patient ID"] = df["Patient ID"].astype(str).str.extract(r"(\d+)")[0].astype(int)
|
||||
df["Diagnosis"] = pd.to_numeric(df["Diagnosis"], errors="coerce")
|
||||
df["VF_MD"] = pd.to_numeric(df["VF_MD"], errors="coerce")
|
||||
# PAPILA encoding: 0=Normal, 1=Glaucoma, 2=Suspect
|
||||
# Binary mode keeps 0 and 1, excludes Suspect (2)
|
||||
df = df[df["Diagnosis"].isin([0, 1])].copy()
|
||||
return df.reset_index(drop=True)
|
||||
|
||||
|
||||
def _build_splits(clinical_dir: Path, fold_seed: int) -> list[Any]:
|
||||
"""Return list of PatientSplit for a given fold seed."""
|
||||
import sys
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
from v3.classes.split_manager import PatientFirstSplitManager, build_patient_split_plans
|
||||
|
||||
df = _load_clinical(clinical_dir)
|
||||
|
||||
# Patient-level label table (mode label per patient)
|
||||
patient_table = (
|
||||
df.groupby(_PATIENT_COL)[_LABEL_COL]
|
||||
.agg(lambda x: x.mode().iloc[0])
|
||||
.reset_index()
|
||||
)
|
||||
plans_raw = build_patient_split_plans(
|
||||
patient_ids=patient_table[_PATIENT_COL].to_numpy(),
|
||||
patient_labels=patient_table[_LABEL_COL].to_numpy(),
|
||||
n_splits=_N_SPLITS,
|
||||
seed=fold_seed,
|
||||
)
|
||||
|
||||
# Wrap into PatientSplit-like objects with .test DataFrame
|
||||
class _Split:
|
||||
def __init__(self, test_ids):
|
||||
self.test = df[df[_PATIENT_COL].isin(test_ids)].reset_index(drop=True)
|
||||
|
||||
return [_Split(p.test_patient_ids) for p in plans_raw]
|
||||
|
||||
|
||||
def get_test_patient_ids(rep: int, fold: int,
|
||||
clinical_dir: Path = CLINICAL_DIR,
|
||||
rep_seed_start: int = _REP_SEED_START,
|
||||
rep_seed_step: int = _REP_SEED_STEP) -> list[int]:
|
||||
"""
|
||||
Return sorted list of Patient IDs in the test set for (rep, fold).
|
||||
Matches the row order of predictions_test.csv for that fold.
|
||||
"""
|
||||
fold_seed = rep_seed_start + rep * rep_seed_step
|
||||
plans = _build_splits(clinical_dir, fold_seed)
|
||||
test_df = plans[fold].test
|
||||
# groupby sorts by default → same order as build_samples / test loader
|
||||
return sorted(test_df[_PATIENT_COL].unique().tolist())
|
||||
|
||||
|
||||
def _row_to_patient_pos(row_idx: int, n_patients: int, batch_size: int) -> int:
|
||||
"""
|
||||
Map a single-mode row index to its patient position in the sorted patient list.
|
||||
|
||||
collect_probs_single_components (aggregate_patient=False) emits predictions
|
||||
in batch-interleaved order: for each batch of B patients, OD rows come first
|
||||
then OS rows. The last batch may be smaller than batch_size.
|
||||
|
||||
Batch i (B patients): rows [i*2B .. i*2B+B-1] = OD
|
||||
[i*2B+B .. i*2B+2B-1] = OS
|
||||
Patient position = i*B + (row_in_batch % B)
|
||||
"""
|
||||
full = n_patients // batch_size
|
||||
last_b = n_patients % batch_size
|
||||
for bi in range(full):
|
||||
s = bi * 2 * batch_size
|
||||
if s <= row_idx < s + 2 * batch_size:
|
||||
return bi * batch_size + (row_idx - s) % batch_size
|
||||
if last_b > 0:
|
||||
s = full * 2 * batch_size
|
||||
return full * batch_size + (row_idx - s) % last_b
|
||||
raise IndexError(f"row_idx {row_idx} out of range for n_patients={n_patients}")
|
||||
|
||||
|
||||
def attach_patient_ids(df: pd.DataFrame,
|
||||
clinical_dir: Path = CLINICAL_DIR,
|
||||
rep_seed_start: int = _REP_SEED_START,
|
||||
rep_seed_step: int = _REP_SEED_STEP) -> pd.DataFrame:
|
||||
"""
|
||||
Add a 'patient_id' column to a pooled predictions DataFrame.
|
||||
Requires 'rep' and 'fold' columns (added by load_all_predictions).
|
||||
The 'idx' column is the row index within each fold's test set.
|
||||
For patient-level modes (ensemble): idx == patient position directly.
|
||||
"""
|
||||
df = df.copy()
|
||||
pid_col = []
|
||||
|
||||
for _, row in df.iterrows():
|
||||
rep_idx = int(row["rep"].replace("rep", ""))
|
||||
fold_idx = int(row["fold"].replace("fold", ""))
|
||||
idx = int(row["idx"])
|
||||
pids = get_test_patient_ids(rep_idx, fold_idx,
|
||||
clinical_dir=clinical_dir,
|
||||
rep_seed_start=rep_seed_start,
|
||||
rep_seed_step=rep_seed_step)
|
||||
pid_col.append(pids[idx] if idx < len(pids) else None)
|
||||
|
||||
df["patient_id"] = pid_col
|
||||
return df
|
||||
|
||||
|
||||
def attach_patient_ids_single(df: pd.DataFrame,
|
||||
clinical_dir: Path = CLINICAL_DIR,
|
||||
batch_size: int = 8,
|
||||
rep_seed_start: int = _REP_SEED_START,
|
||||
rep_seed_step: int = _REP_SEED_STEP) -> pd.DataFrame:
|
||||
"""
|
||||
Like attach_patient_ids but for single (eye-level) mode.
|
||||
Single mode emits predictions in batch-interleaved order (see _row_to_patient_pos).
|
||||
batch_size must match the --batch-size used during training (default 8).
|
||||
"""
|
||||
df = df.copy()
|
||||
pid_col = []
|
||||
|
||||
for _, row in df.iterrows():
|
||||
rep_idx = int(row["rep"].replace("rep", ""))
|
||||
fold_idx = int(row["fold"].replace("fold", ""))
|
||||
idx = int(row["idx"])
|
||||
pids = get_test_patient_ids(rep_idx, fold_idx,
|
||||
clinical_dir=clinical_dir,
|
||||
rep_seed_start=rep_seed_start,
|
||||
rep_seed_step=rep_seed_step)
|
||||
patient_pos = _row_to_patient_pos(idx, len(pids), batch_size)
|
||||
pid_col.append(pids[patient_pos])
|
||||
|
||||
df["patient_id"] = pid_col
|
||||
return df
|
||||
@@ -0,0 +1,569 @@
|
||||
"""
|
||||
GradCAM analysis for Phase 5 — logit_mlp_head checkpointed run.
|
||||
|
||||
Produces (all in figures/explainability/gradcam/):
|
||||
mean_cam_normal.png — average heatmap across all normal test eyes
|
||||
mean_cam_glaucoma.png — average heatmap across all glaucoma test eyes
|
||||
mean_cam_comparison.png — side-by-side normal vs glaucoma mean CAMs
|
||||
overlay_grid_normal.png — grid of individual overlays (normal eyes)
|
||||
overlay_grid_glaucoma.png — grid of individual overlays (glaucoma eyes)
|
||||
|
||||
Checkpoints loaded from:
|
||||
v3/results/phase5/logit_mlp_head_ckpt/rep00/binary/ensemble/fold{0..4}/best_single.pt
|
||||
|
||||
Usage:
|
||||
python -m v3.scripts.output_analysis.explainability.gradcam_phase5
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.cm as cm
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from PIL import Image
|
||||
from tqdm import tqdm
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
CKPT_RUN = REPO_ROOT / "v3" / "results" / "phase5" / "logit_mlp_head_ckpt"
|
||||
FIGURES_ROOT = REPO_ROOT / "v3" / "figures" / "explainability" / "gradcam"
|
||||
CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
|
||||
IMAGE_DIR = REPO_ROOT / "Papila" / "FundusImages"
|
||||
CONTOUR_DIR = REPO_ROOT / "Papila" / "ExpertsSegmentations" / "Contours"
|
||||
|
||||
DISC_SPAN = 5 # patch side = DISC_SPAN × disc diameter
|
||||
PATCH_SIZE = 96 # output thumbnail pixels
|
||||
|
||||
# Model hyperparameters (inferred from checkpoint weight shapes)
|
||||
BACKBONE = "resnet50"
|
||||
NUM_CLASSES = 2
|
||||
CD_HIDDEN = 128
|
||||
FUSION_DIM = 256
|
||||
|
||||
LABEL_NAMES = {0: "Normal", 1: "Glaucoma"}
|
||||
|
||||
|
||||
# ── GradCAM ──────────────────────────────────────────────────────────────────
|
||||
|
||||
class GradCAM:
|
||||
"""Minimal GradCAM via forward/backward hooks."""
|
||||
|
||||
def __init__(self, target_layer: torch.nn.Module) -> None:
|
||||
self._acts = None
|
||||
self._grads = None
|
||||
self._h1 = target_layer.register_forward_hook(self._save_acts)
|
||||
self._h2 = target_layer.register_full_backward_hook(self._save_grads)
|
||||
|
||||
def _save_acts(self, _m, _i, output):
|
||||
self._acts = output.detach()
|
||||
|
||||
def _save_grads(self, _m, _gi, grad_output):
|
||||
self._grads = grad_output[0].detach()
|
||||
|
||||
def compute(self, img: torch.Tensor, meta: torch.Tensor,
|
||||
model: torch.nn.Module, target_class: int | None = None) -> tuple[np.ndarray, int]:
|
||||
"""Return (cam [H,W] normalised 0-1, predicted_class)."""
|
||||
model.eval()
|
||||
with torch.enable_grad():
|
||||
out = model(img, meta)
|
||||
pred = int(out.argmax(1).item())
|
||||
tc = pred if target_class is None else target_class
|
||||
model.zero_grad()
|
||||
out[0, tc].backward()
|
||||
|
||||
weights = self._grads.mean(dim=(2, 3), keepdim=True)
|
||||
cam = F.relu((weights * self._acts).sum(dim=1, keepdim=True))
|
||||
cam = F.interpolate(cam, img.shape[-2:], mode="bilinear", align_corners=False)
|
||||
cam_np = cam.squeeze().cpu().numpy()
|
||||
lo, hi = cam_np.min(), cam_np.max()
|
||||
return (cam_np - lo) / (hi - lo + 1e-8), pred
|
||||
|
||||
def remove(self) -> None:
|
||||
self._h1.remove(); self._h2.remove()
|
||||
|
||||
|
||||
def overlay_gradcam(pil: Image.Image, cam: np.ndarray, alpha: float = 0.45) -> Image.Image:
|
||||
cam_u8 = (cam * 255).astype(np.uint8)
|
||||
cam_r = np.array(Image.fromarray(cam_u8).resize(pil.size, Image.BILINEAR)) / 255.0
|
||||
colored = (cm.jet(cam_r)[:, :, :3] * 255).astype(np.uint8)
|
||||
return Image.blend(pil.convert("RGB"), Image.fromarray(colored), alpha)
|
||||
|
||||
|
||||
# ── Disc-centred attention helpers ────────────────────────────────────────────
|
||||
|
||||
def _disc_contour_path(pid: int, eye: str, expert: int = 1) -> Path:
|
||||
return CONTOUR_DIR / f"RET{pid:03d}{eye}_disc_exp{expert}.txt"
|
||||
|
||||
|
||||
def _load_disc_mask(pid: int, eye: str, cam_h: int, cam_w: int) -> np.ndarray | None:
|
||||
"""Load expert disc contour, polygon-fill, resize to (cam_h, cam_w)."""
|
||||
from PIL import ImageDraw as _ID
|
||||
p = _disc_contour_path(pid, eye)
|
||||
if not p.exists():
|
||||
return None
|
||||
try:
|
||||
arr = np.loadtxt(str(p), dtype=np.float32)
|
||||
except Exception:
|
||||
return None
|
||||
if arr.ndim == 1:
|
||||
arr = arr.reshape(-1, 2)
|
||||
if arr.shape[0] < 3:
|
||||
return None
|
||||
# Get original image size
|
||||
img_path = get_image_path(pid, eye)
|
||||
try:
|
||||
with Image.open(img_path) as im:
|
||||
orig_w, orig_h = im.size
|
||||
except Exception:
|
||||
return None
|
||||
canvas = Image.new("L", (orig_w, orig_h), 0)
|
||||
_ID.Draw(canvas).polygon([tuple(pt) for pt in arr[:, :2]], fill=1)
|
||||
return np.array(canvas.resize((cam_w, cam_h), Image.NEAREST), dtype=bool)
|
||||
|
||||
|
||||
def _disc_centred_patch(cam: np.ndarray, disc_mask: np.ndarray,
|
||||
span: int = DISC_SPAN, out: int = PATCH_SIZE
|
||||
) -> tuple[np.ndarray | None, float | None]:
|
||||
"""Translate+scale cam so disc centroid is centred; return (patch, disc_r_out)."""
|
||||
if disc_mask is None or disc_mask.sum() == 0:
|
||||
return None, None
|
||||
ys, xs = np.where(disc_mask)
|
||||
cy, cx = ys.mean(), xs.mean()
|
||||
disc_r = float(np.sqrt(disc_mask.sum() / np.pi))
|
||||
half = max(1, int(round(span * disc_r / 2)))
|
||||
h, w = cam.shape
|
||||
y0, y1 = int(round(cy)) - half, int(round(cy)) + half
|
||||
x0, x1 = int(round(cx)) - half, int(round(cx)) + half
|
||||
pt = max(0, -y0); pb = max(0, y1 - h)
|
||||
pl = max(0, -x0); pr = max(0, x1 - w)
|
||||
cam_pad = np.pad(cam, ((pt, pb), (pl, pr)), constant_values=0.0)
|
||||
patch = cam_pad[y0 + pt: y1 + pt, x0 + pl: x1 + pl]
|
||||
patch_out = np.array(
|
||||
Image.fromarray((np.clip(patch, 0, 1) * 255).astype(np.uint8))
|
||||
.resize((out, out), Image.BILINEAR)
|
||||
) / 255.0
|
||||
disc_r_out = out * disc_r / (2 * half)
|
||||
return patch_out.astype(np.float32), disc_r_out
|
||||
|
||||
|
||||
def make_disc_attention_detail(
|
||||
mean_patches: dict,
|
||||
stats_rows: list[dict],
|
||||
out_path: Path,
|
||||
) -> None: # noqa: C901
|
||||
"""
|
||||
2-row (Normal / Glaucoma) × 3-col (correct cam | incorrect cam | disc_frac strip).
|
||||
|
||||
mean_patches: {(cls_name, split): (mean_patch_array, mean_disc_r, count)}
|
||||
stats_rows: list of {true_name, correct, disc_frac} dicts (floats only, no arrays)
|
||||
"""
|
||||
import pandas as pd
|
||||
from matplotlib.patches import Circle
|
||||
|
||||
classes = ["Normal", "Glaucoma"]
|
||||
splits = ["correct", "incorrect"]
|
||||
corr_colors = {"correct": "steelblue", "incorrect": "tomato"}
|
||||
stats = pd.DataFrame(stats_rows)
|
||||
|
||||
# 2 rows (Normal / Glaucoma) × 3 cols (correct cam | incorrect cam | disc_frac strip)
|
||||
fig, axes = plt.subplots(2, 3, figsize=(13, 8),
|
||||
gridspec_kw={"width_ratios": [1, 1, 0.75]})
|
||||
fig.patch.set_facecolor("#f4f4f4")
|
||||
|
||||
rng = np.random.default_rng(42)
|
||||
|
||||
for ri, cls in enumerate(classes):
|
||||
# Col 0 & 1: correct / incorrect mean CAMs
|
||||
for ci, split in enumerate(splits):
|
||||
ax = axes[ri, ci]
|
||||
ax.set_facecolor("#222")
|
||||
key = (cls, split)
|
||||
if key in mean_patches:
|
||||
mp, disc_r_out, count = mean_patches[key]
|
||||
ax.imshow(mp, cmap="jet", vmin=0, vmax=1, origin="upper",
|
||||
extent=[0, PATCH_SIZE, PATCH_SIZE, 0])
|
||||
cx = cy = PATCH_SIZE / 2
|
||||
ax.add_patch(Circle((cx, cy), disc_r_out,
|
||||
fill=False, edgecolor="white",
|
||||
linewidth=2, linestyle="--"))
|
||||
ax.set_title(f"{split.capitalize()} (N={count})", fontsize=9)
|
||||
else:
|
||||
ax.text(0.5, 0.5, "no data", ha="center", va="center",
|
||||
transform=ax.transAxes, fontsize=9, color="grey")
|
||||
ax.set_title(split.capitalize(), fontsize=9)
|
||||
ax.axis("off")
|
||||
|
||||
# Row label on leftmost column
|
||||
axes[ri, 0].set_ylabel(cls, fontsize=11, fontweight="bold", labelpad=8)
|
||||
|
||||
# Col 2: disc_frac strip
|
||||
ax = axes[ri, 2]
|
||||
ax.set_facecolor("#f4f4f4")
|
||||
sub = stats[stats["true_name"] == cls].dropna(subset=["disc_frac"])
|
||||
for xi, split in enumerate(splits):
|
||||
pts = sub[sub["correct"] == (split == "correct")]["disc_frac"].values
|
||||
if len(pts) == 0:
|
||||
continue
|
||||
color = corr_colors[split]
|
||||
jitter = rng.uniform(-0.18, 0.18, size=len(pts))
|
||||
ax.scatter(xi + jitter, pts, color=color, alpha=0.7, s=28, edgecolors="none")
|
||||
ax.hlines(pts.mean(), xi - 0.28, xi + 0.28, colors=color, linewidth=2.5, zorder=5)
|
||||
ax.set_xticks([0, 1])
|
||||
ax.set_xticklabels(["Correct", "Incorrect"], fontsize=9)
|
||||
ax.set_xlim(-0.55, 1.55)
|
||||
ax.set_ylim(0, 1)
|
||||
ax.set_title("Disc fraction", fontsize=9)
|
||||
ax.grid(axis="y", linestyle="--", alpha=0.3)
|
||||
if ri == 0:
|
||||
ax.set_ylabel("Attention mass inside GT disc", fontsize=9)
|
||||
|
||||
fig.suptitle(
|
||||
"Disc-centred GradCAM attention | dashed circle = GT disc boundary\n"
|
||||
"Phase 5, logit_mlp_head, fold 0–4",
|
||||
fontsize=11, fontweight="bold",
|
||||
)
|
||||
fig.tight_layout()
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out_path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {out_path}")
|
||||
|
||||
|
||||
# ── Model loading ─────────────────────────────────────────────────────────────
|
||||
|
||||
def build_model(ckpt_path: Path, device: torch.device):
|
||||
"""Reconstruct SingleEyeHT from checkpoint and load weights."""
|
||||
from types import SimpleNamespace
|
||||
from v3.classes.models import SingleEyeHT
|
||||
sd = torch.load(ckpt_path, map_location="cpu")
|
||||
# ClinicalTower only reads clinical_data.feature_dim at init time
|
||||
cd_in = sd["cd_tower.block0.0.weight"].shape[1]
|
||||
clinical_shim = SimpleNamespace(feature_dim=cd_in)
|
||||
model = SingleEyeHT(
|
||||
backbone=BACKBONE,
|
||||
freeze_ratio=0.0,
|
||||
augment=False,
|
||||
clinical_data=clinical_shim,
|
||||
num_classes=NUM_CLASSES,
|
||||
cd_hidden_dim=CD_HIDDEN,
|
||||
fusion_dim=FUSION_DIM,
|
||||
)
|
||||
model.load_state_dict(sd)
|
||||
model.to(device).eval()
|
||||
return model
|
||||
|
||||
|
||||
# ── Data helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
def build_data_bundle():
|
||||
"""Build the PAPILA DataBundle matching the checkpointed run's feature config."""
|
||||
from v3.classes.papila_builders import build_papila_data
|
||||
import torch as _t
|
||||
# Auto-detect cd_in from the majority of checkpoints (excludes stale reps).
|
||||
import collections as _col
|
||||
all_ckpts = list(CKPT_RUN.glob("rep*/binary/ensemble/fold*/best_single.pt"))
|
||||
if all_ckpts:
|
||||
counts = _col.Counter(
|
||||
_t.load(c, map_location="cpu")["cd_tower.block0.0.weight"].shape[1]
|
||||
for c in all_ckpts
|
||||
)
|
||||
cd_in = counts.most_common(1)[0][0]
|
||||
else:
|
||||
cd_in = 25
|
||||
drop_raw = cd_in <= 21
|
||||
excl = ["Axial_Length"] if cd_in in (21, 23) else []
|
||||
return build_papila_data(
|
||||
image_dir=str(IMAGE_DIR),
|
||||
clinical_dir=str(CLINICAL_DIR),
|
||||
label_col="Diagnosis",
|
||||
cat_cols=["Gender", "Phakic/Pseudophakic"],
|
||||
iop_corr_method="ratio",
|
||||
iop_drop_raw=drop_raw,
|
||||
exclude_cols=excl,
|
||||
)
|
||||
|
||||
|
||||
def get_image_path(pid: int, eye: str) -> Path:
|
||||
return IMAGE_DIR / f"RET{pid:03d}{eye}.jpg"
|
||||
|
||||
|
||||
def build_meta_vector(row, data) -> torch.Tensor:
|
||||
"""Build the training-compatible feature vector via DataBundle.vectorize_row."""
|
||||
vec = data.vectorize_row(row)
|
||||
return torch.tensor(vec, dtype=torch.float32).unsqueeze(0)
|
||||
|
||||
|
||||
# ── Eval transform ────────────────────────────────────────────────────────────
|
||||
|
||||
def get_eval_transform():
|
||||
from torchvision import transforms
|
||||
return transforms.Compose([
|
||||
transforms.Resize(256),
|
||||
transforms.CenterCrop(224),
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize(mean=(0.485, 0.456, 0.406),
|
||||
std=(0.229, 0.224, 0.225)),
|
||||
])
|
||||
|
||||
|
||||
# ── Main loop ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def _discover_checkpoints(ckpt_run: Path) -> list[tuple[int, int, Path]]:
|
||||
"""
|
||||
Scan ckpt_run for all available best_single.pt files.
|
||||
Skips checkpoints whose cd_in doesn't match the majority (to exclude stale reps).
|
||||
Returns sorted list of (rep_idx, fold_idx, ckpt_path).
|
||||
"""
|
||||
import collections
|
||||
candidates = []
|
||||
for rep_dir in sorted(ckpt_run.glob("rep*")):
|
||||
try:
|
||||
rep_idx = int(rep_dir.name.replace("rep", ""))
|
||||
except ValueError:
|
||||
continue
|
||||
for fold_dir in sorted((rep_dir / "binary" / "ensemble").glob("fold[0-9]")):
|
||||
ckpt = fold_dir / "best_single.pt"
|
||||
if ckpt.exists():
|
||||
fold_idx = int(fold_dir.name.replace("fold", ""))
|
||||
cd_in = torch.load(ckpt, map_location="cpu")[
|
||||
"cd_tower.block0.0.weight"
|
||||
].shape[1]
|
||||
candidates.append((rep_idx, fold_idx, ckpt, cd_in))
|
||||
|
||||
if not candidates:
|
||||
return []
|
||||
|
||||
# Use the majority cd_in so stale reps are automatically excluded
|
||||
counts = collections.Counter(c[3] for c in candidates)
|
||||
target_cd_in = counts.most_common(1)[0][0]
|
||||
skipped = sum(1 for c in candidates if c[3] != target_cd_in)
|
||||
if skipped:
|
||||
print(f" [discover] skipping {skipped} checkpoint(s) with cd_in≠{target_cd_in}")
|
||||
|
||||
return [(rep, fold, ckpt) for rep, fold, ckpt, cd in candidates if cd == target_cd_in]
|
||||
|
||||
|
||||
def run(n_grid: int = 16, alpha: float = 0.45, target_class: int | None = None):
|
||||
"""
|
||||
Loop over all available checkpoints in the run directory (all reps × folds).
|
||||
Aggregate CAMs per class, collect overlay grids.
|
||||
"""
|
||||
import pandas as pd
|
||||
from v3.scripts.output_analysis.explainability.fold_patient_ids import (
|
||||
get_test_patient_ids,
|
||||
)
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
print(f"Device: {device}")
|
||||
|
||||
print("Building DataBundle ...")
|
||||
data = build_data_bundle()
|
||||
clinical = data.df
|
||||
print(f" feature_dim={data.feature_dim} rows={len(clinical)}")
|
||||
transform = get_eval_transform()
|
||||
|
||||
checkpoints = _discover_checkpoints(CKPT_RUN)
|
||||
print(f"Found {len(checkpoints)} checkpoint(s) across "
|
||||
f"{len(set(r for r,f,_ in checkpoints))} rep(s)")
|
||||
|
||||
if not checkpoints:
|
||||
print("No checkpoints found — run with --save-checkpoints first.")
|
||||
return
|
||||
|
||||
# ── Incremental accumulators (no full-res arrays kept after each eye) ────────
|
||||
# Mean CAM per class: running sum
|
||||
cam_sum = {0: None, 1: None}
|
||||
cam_count = {0: 0, 1: 0}
|
||||
|
||||
# Overlay grid: keep at most n_grid PIL images per class (capped)
|
||||
overlay_items = {0: [], 1: []}
|
||||
|
||||
# Disc-attention detail: running sum of disc patches (not list of arrays)
|
||||
disc_patch_sum = {} # (cls_name, split) → np.ndarray sum
|
||||
disc_patch_count = {} # (cls_name, split) → int
|
||||
disc_radius_sum = {} # (cls_name, split) → float sum
|
||||
disc_stats_rows = [] # floats only — no arrays
|
||||
|
||||
ckpt_bar = tqdm(checkpoints, desc="Folds", unit="fold")
|
||||
for rep_idx, fold_idx, ckpt_path in ckpt_bar:
|
||||
ckpt_bar.set_postfix(rep=rep_idx, fold=fold_idx)
|
||||
model = build_model(ckpt_path, device)
|
||||
|
||||
# GradCAM target: last ResNet block
|
||||
target_layer = model.img_tower.backbone.layer4[-1]
|
||||
gcam = GradCAM(target_layer)
|
||||
|
||||
pids = get_test_patient_ids(rep_idx, fold_idx, clinical_dir=CLINICAL_DIR)
|
||||
|
||||
for pid in tqdm(pids, desc=f" rep{rep_idx:02d}/fold{fold_idx}", leave=False, unit="pt"):
|
||||
for eye in ("OD", "OS"):
|
||||
img_path = get_image_path(pid, eye)
|
||||
if not img_path.exists():
|
||||
continue
|
||||
|
||||
row = clinical[
|
||||
(clinical["Patient ID"] == pid) & (clinical["eyeID"] == eye)
|
||||
]
|
||||
if len(row) == 0:
|
||||
continue
|
||||
row = row.iloc[0]
|
||||
label = int(row["Diagnosis"])
|
||||
|
||||
pil_orig = Image.open(img_path).convert("RGB")
|
||||
img_t = transform(pil_orig).unsqueeze(0).to(device)
|
||||
meta_t = build_meta_vector(row, data).to(device)
|
||||
|
||||
cam_np, pred = gcam.compute(img_t, meta_t, model,
|
||||
target_class=target_class)
|
||||
|
||||
# Running mean CAM
|
||||
if cam_sum[label] is None:
|
||||
cam_sum[label] = cam_np.copy()
|
||||
else:
|
||||
cam_sum[label] += cam_np
|
||||
cam_count[label] += 1
|
||||
|
||||
# Overlay grid — only keep up to n_grid per class
|
||||
if len(overlay_items[label]) < n_grid:
|
||||
ov = overlay_gradcam(pil_orig, cam_np, alpha=alpha)
|
||||
overlay_items[label].append((ov, pid, eye, pred))
|
||||
|
||||
# Disc-attention: extract patch now, accumulate into running sum
|
||||
h, w = cam_np.shape
|
||||
disc_mask = _load_disc_mask(pid, eye, h, w)
|
||||
disc_frac = None
|
||||
if disc_mask is not None and disc_mask.sum() > 0:
|
||||
disc_frac = float(cam_np[disc_mask].sum() / (cam_np.sum() + 1e-8))
|
||||
|
||||
cls_name = LABEL_NAMES[label]
|
||||
split = "correct" if (pred == label) else "incorrect"
|
||||
key = (cls_name, split)
|
||||
|
||||
patch, disc_r_out = _disc_centred_patch(cam_np, disc_mask)
|
||||
if patch is not None:
|
||||
if key not in disc_patch_sum:
|
||||
disc_patch_sum[key] = patch.copy()
|
||||
disc_patch_count[key] = 1
|
||||
disc_radius_sum[key] = disc_r_out
|
||||
else:
|
||||
disc_patch_sum[key] += patch
|
||||
disc_patch_count[key] += 1
|
||||
disc_radius_sum[key] += disc_r_out
|
||||
|
||||
disc_stats_rows.append({
|
||||
"true_name": cls_name,
|
||||
"correct": pred == label,
|
||||
"disc_frac": disc_frac,
|
||||
})
|
||||
|
||||
# Release per-eye tensors immediately
|
||||
del img_t, meta_t, cam_np
|
||||
if disc_mask is not None:
|
||||
del disc_mask
|
||||
|
||||
gcam.remove()
|
||||
del model
|
||||
torch.cuda.empty_cache() if torch.cuda.is_available() else None
|
||||
|
||||
# Build mean_patches dict for disc detail plot
|
||||
mean_patches = {
|
||||
key: (
|
||||
disc_patch_sum[key] / disc_patch_count[key],
|
||||
disc_radius_sum[key] / disc_patch_count[key],
|
||||
disc_patch_count[key],
|
||||
)
|
||||
for key in disc_patch_sum
|
||||
}
|
||||
|
||||
# ── Save outputs ─────────────────────────────────────────────────────────
|
||||
FIGURES_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for cls in [0, 1]:
|
||||
if cam_count[cls] == 0:
|
||||
continue
|
||||
mean_cam = cam_sum[cls] / cam_count[cls]
|
||||
lo, hi = mean_cam.min(), mean_cam.max()
|
||||
mean_cam = (mean_cam - lo) / (hi - lo + 1e-8)
|
||||
|
||||
fig, ax = plt.subplots(figsize=(5, 5))
|
||||
ax.imshow(mean_cam, cmap="jet", vmin=0, vmax=1)
|
||||
ax.axis("off")
|
||||
ax.set_title(f"Mean GradCAM — {LABEL_NAMES[cls]}\n(n={cam_count[cls]} eyes, fold 0–4)",
|
||||
fontsize=11, fontweight="bold")
|
||||
plt.colorbar(ax.images[0], ax=ax, fraction=0.046, pad=0.04)
|
||||
out = FIGURES_ROOT / f"mean_cam_{LABEL_NAMES[cls].lower()}.png"
|
||||
fig.savefig(out, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {out}")
|
||||
|
||||
# Side-by-side comparison
|
||||
if cam_count[0] > 0 and cam_count[1] > 0:
|
||||
fig, axes = plt.subplots(1, 2, figsize=(10, 5))
|
||||
fig.suptitle("Mean GradCAM — Normal vs Glaucoma (Phase 5, fold 0–4)",
|
||||
fontsize=12, fontweight="bold")
|
||||
for ax, cls in zip(axes, [0, 1]):
|
||||
mean_cam = cam_sum[cls] / cam_count[cls]
|
||||
lo, hi = mean_cam.min(), mean_cam.max()
|
||||
mean_cam = (mean_cam - lo) / (hi - lo + 1e-8)
|
||||
im = ax.imshow(mean_cam, cmap="jet", vmin=0, vmax=1)
|
||||
ax.axis("off")
|
||||
ax.set_title(f"{LABEL_NAMES[cls]} (n={cam_count[cls]})", fontsize=11)
|
||||
plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
|
||||
out = FIGURES_ROOT / "mean_cam_comparison.png"
|
||||
fig.savefig(out, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {out}")
|
||||
|
||||
# Overlay grids
|
||||
for cls in [0, 1]:
|
||||
items = overlay_items[cls]
|
||||
if not items:
|
||||
continue
|
||||
# Sort: misclassified first (more interesting)
|
||||
items.sort(key=lambda x: x[3] == cls) # wrong preds first
|
||||
items = items[:n_grid]
|
||||
ncols = 4
|
||||
nrows = int(np.ceil(len(items) / ncols))
|
||||
fig, axes = plt.subplots(nrows, ncols, figsize=(ncols * 3.2, nrows * 3.2))
|
||||
axes = np.array(axes).reshape(-1)
|
||||
fig.suptitle(f"GradCAM Overlays — {LABEL_NAMES[cls]} (Phase 5)",
|
||||
fontsize=12, fontweight="bold")
|
||||
for i, ax in enumerate(axes):
|
||||
if i < len(items):
|
||||
ov, pid, eye, pred = items[i]
|
||||
ax.imshow(ov)
|
||||
correct = pred == cls
|
||||
col = "#2e7d32" if correct else "#c62828"
|
||||
ax.set_title(f"RET{pid:03d}{eye}\n→ {LABEL_NAMES[pred]}",
|
||||
fontsize=7.5, color=col)
|
||||
ax.axis("off")
|
||||
out = FIGURES_ROOT / f"overlay_grid_{LABEL_NAMES[cls].lower()}.png"
|
||||
fig.tight_layout()
|
||||
fig.savefig(out, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {out}")
|
||||
|
||||
|
||||
# Disc-centred detail plot
|
||||
if disc_patch_sum:
|
||||
make_disc_attention_detail(mean_patches, disc_stats_rows,
|
||||
FIGURES_ROOT / "disc_attention_detail.png")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--n-grid", type=int, default=16,
|
||||
help="Max overlays per class in grid (default 16)")
|
||||
ap.add_argument("--alpha", type=float, default=0.45,
|
||||
help="GradCAM overlay opacity (default 0.45)")
|
||||
ap.add_argument("--target-class", type=int, default=None,
|
||||
help="GradCAM target class (default: predicted class)")
|
||||
args = ap.parse_args()
|
||||
run(n_grid=args.n_grid, alpha=args.alpha, target_class=args.target_class)
|
||||
@@ -0,0 +1,309 @@
|
||||
"""
|
||||
MD permutation feature importance for Phase 5 — logit_mlp_head checkpointed run.
|
||||
|
||||
For each of the 5 fold checkpoints:
|
||||
- loads test images + clinical metadata
|
||||
- caches image features (no grad)
|
||||
- permutes each clinical feature N times and measures AUC drop
|
||||
|
||||
Produces (in figures/explainability/):
|
||||
md_importance_phase5.png — aggregated bar chart across 5 folds
|
||||
md_importance_phase5.csv — mean/std per feature
|
||||
|
||||
Usage:
|
||||
python -m v3.scripts.output_analysis.explainability.permutation_importance_phase5
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from sklearn.metrics import roc_auc_score
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
CKPT_RUN = REPO_ROOT / "v3" / "results" / "phase5" / "logit_mlp_head_ckpt"
|
||||
FIGURES_ROOT = REPO_ROOT / "v3" / "figures" / "explainability"
|
||||
CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
|
||||
IMAGE_DIR = REPO_ROOT / "Papila" / "FundusImages"
|
||||
|
||||
BACKBONE = "resnet50"
|
||||
NUM_CLASSES = 2
|
||||
CD_HIDDEN = 128
|
||||
FUSION_DIM = 256
|
||||
N_PERMUTATIONS = 30
|
||||
SEED = 0
|
||||
|
||||
|
||||
# ── Model ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def build_model(ckpt_path: Path, device: torch.device):
|
||||
from v3.classes.models import SingleEyeHT
|
||||
sd = torch.load(ckpt_path, map_location="cpu")
|
||||
cd_in = sd["cd_tower.block0.0.weight"].shape[1]
|
||||
model = SingleEyeHT(
|
||||
backbone=BACKBONE, freeze_ratio=0.0, augment=False,
|
||||
clinical_data=SimpleNamespace(feature_dim=cd_in),
|
||||
num_classes=NUM_CLASSES, cd_hidden_dim=CD_HIDDEN, fusion_dim=FUSION_DIM,
|
||||
)
|
||||
model.load_state_dict(sd)
|
||||
model.to(device).eval()
|
||||
return model
|
||||
|
||||
|
||||
# ── Data ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def build_data_bundle():
|
||||
from v3.classes.papila_builders import build_papila_data
|
||||
# Infer settings from available checkpoint to stay compatible.
|
||||
# Once the 10x5 run (--iop-drop-raw --exclude-cols Axial_Length) completes,
|
||||
# these will automatically match (feature_dim will drop from 25 → 21).
|
||||
ckpt = next(CKPT_RUN.glob("rep*/binary/ensemble/fold*/best_single.pt"), None)
|
||||
import torch as _t
|
||||
cd_in = _t.load(ckpt, map_location="cpu")["cd_tower.block0.0.weight"].shape[1] if ckpt else 25
|
||||
# cd_in=25 → old run (no iop_drop_raw, no excl); cd_in=21 → new run
|
||||
drop_raw = cd_in <= 21
|
||||
excl = ["Axial_Length"] if cd_in in (21, 23) else []
|
||||
return build_papila_data(
|
||||
image_dir=str(IMAGE_DIR), clinical_dir=str(CLINICAL_DIR),
|
||||
label_col="Diagnosis", cat_cols=["Gender", "Phakic/Pseudophakic"],
|
||||
iop_corr_method="ratio", iop_drop_raw=drop_raw, exclude_cols=excl,
|
||||
)
|
||||
|
||||
|
||||
def get_eval_transform():
|
||||
from torchvision import transforms
|
||||
return transforms.Compose([
|
||||
transforms.Resize(256), transforms.CenterCrop(224),
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
|
||||
])
|
||||
|
||||
|
||||
def get_image_path(pid: int, eye: str) -> Path:
|
||||
return IMAGE_DIR / f"RET{pid:03d}{eye}.jpg"
|
||||
|
||||
|
||||
# ── Feature index map ─────────────────────────────────────────────────────────
|
||||
|
||||
def build_feature_index_map(data) -> dict[str, list[int]]:
|
||||
"""
|
||||
Map feature name → list of dimension indices in the vectorize_row output.
|
||||
Layout: [scalars (min-max scaled)] + [cat one-hots] + [scalar missing flags]
|
||||
"""
|
||||
n_scalar = len(data.scalar_cols)
|
||||
cat_expanded = sum(len(m) for m in data.cat_maps.values())
|
||||
feat_map: dict[str, list[int]] = {}
|
||||
|
||||
# Scalar: value dim + missing flag dim
|
||||
for i, col in enumerate(data.scalar_cols):
|
||||
feat_map[col] = [i, n_scalar + cat_expanded + i]
|
||||
|
||||
# Categorical: whole one-hot block
|
||||
cat_offset = n_scalar
|
||||
for col in data.cat_cols:
|
||||
n = len(data.cat_maps[col])
|
||||
feat_map[col] = list(range(cat_offset, cat_offset + n))
|
||||
cat_offset += n
|
||||
|
||||
return feat_map
|
||||
|
||||
|
||||
# ── Per-fold importance ───────────────────────────────────────────────────────
|
||||
|
||||
def run_fold(rep_idx: int, fold_idx: int, model, data, device: torch.device,
|
||||
n_permutations: int, seed: int) -> dict[str, tuple[float, float]]:
|
||||
"""
|
||||
Returns {feature_name: (mean_auc_drop, std_auc_drop)}.
|
||||
"""
|
||||
from v3.scripts.output_analysis.explainability.fold_patient_ids import (
|
||||
get_test_patient_ids,
|
||||
)
|
||||
transform = get_eval_transform()
|
||||
clinical = data.df
|
||||
|
||||
pids = get_test_patient_ids(rep_idx, fold_idx, clinical_dir=CLINICAL_DIR)
|
||||
|
||||
# Cache image features + build meta tensors + labels
|
||||
img_feats_list, meta_list, label_list = [], [], []
|
||||
model.eval()
|
||||
with torch.no_grad():
|
||||
for pid in pids:
|
||||
for eye in ("OD", "OS"):
|
||||
img_path = get_image_path(pid, eye)
|
||||
if not img_path.exists():
|
||||
continue
|
||||
row = clinical[
|
||||
(clinical["Patient ID"] == pid) & (clinical["eyeID"] == eye)
|
||||
]
|
||||
if len(row) == 0:
|
||||
continue
|
||||
row = row.iloc[0]
|
||||
label = int(row["Diagnosis"])
|
||||
|
||||
from PIL import Image
|
||||
pil = Image.open(img_path).convert("RGB")
|
||||
img_t = transform(pil).unsqueeze(0).to(device)
|
||||
feats = model.img_tower(img_t) # [1, img_dim]
|
||||
meta_vec = torch.tensor(data.vectorize_row(row),
|
||||
dtype=torch.float32).unsqueeze(0)
|
||||
|
||||
img_feats_list.append(feats.cpu())
|
||||
meta_list.append(meta_vec)
|
||||
label_list.append(label)
|
||||
|
||||
if not label_list or len(set(label_list)) < 2:
|
||||
print(f" fold{fold_idx}: insufficient data, skipping.")
|
||||
return {}
|
||||
|
||||
img_feats = torch.cat(img_feats_list).to(device) # [N, img_dim]
|
||||
meta_all = torch.cat(meta_list) # [N, feat_dim] on CPU
|
||||
y_true = np.array(label_list)
|
||||
|
||||
# Baseline AUC
|
||||
with torch.no_grad():
|
||||
md_feats = model.cd_tower(meta_all.to(device))
|
||||
out_f, _, _ = model.bridge(img_feats, md_feats)
|
||||
probs_base = F.softmax(out_f, dim=1)[:, 1].cpu().numpy()
|
||||
baseline_auc = roc_auc_score(y_true, probs_base)
|
||||
print(f" fold{fold_idx}: baseline AUC={baseline_auc:.4f} N={len(y_true)}")
|
||||
|
||||
feat_map = build_feature_index_map(data)
|
||||
rng = np.random.default_rng(seed + fold_idx)
|
||||
results: dict[str, tuple[float, float]] = {}
|
||||
|
||||
for feat_name, dims in feat_map.items():
|
||||
drops = []
|
||||
for _ in range(n_permutations):
|
||||
meta_perm = meta_all.clone()
|
||||
perm_idx = rng.permutation(len(meta_perm))
|
||||
meta_perm[:, dims] = meta_perm[perm_idx][:, dims]
|
||||
with torch.no_grad():
|
||||
md_p = model.cd_tower(meta_perm.to(device))
|
||||
out_p, _, _ = model.bridge(img_feats, md_p)
|
||||
probs_p = F.softmax(out_p, dim=1)[:, 1].cpu().numpy()
|
||||
try:
|
||||
drops.append(baseline_auc - roc_auc_score(y_true, probs_p))
|
||||
except Exception:
|
||||
pass
|
||||
if drops:
|
||||
results[feat_name] = (float(np.mean(drops)), float(np.std(drops)))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# ── Aggregate and plot ────────────────────────────────────────────────────────
|
||||
|
||||
def plot_importance(all_results: list[dict], out_png: Path, out_csv: Path) -> None:
|
||||
# Aggregate across folds
|
||||
all_feats = sorted({f for r in all_results for f in r})
|
||||
agg = {}
|
||||
for feat in all_feats:
|
||||
vals = [r[feat][0] for r in all_results if feat in r]
|
||||
if vals:
|
||||
agg[feat] = (float(np.mean(vals)), float(np.std(vals)))
|
||||
|
||||
# Sort by mean importance descending
|
||||
sorted_feats = sorted(agg, key=lambda f: agg[f][0], reverse=True)
|
||||
names = sorted_feats
|
||||
imps = [agg[f][0] for f in names]
|
||||
stds = [agg[f][1] for f in names]
|
||||
colors = ["#e05c5c" if v >= 0 else "#5c9ee0" for v in imps]
|
||||
|
||||
fig, ax = plt.subplots(figsize=(9, max(4, len(names) * 0.45 + 1.5)))
|
||||
y_pos = np.arange(len(names))
|
||||
ax.barh(y_pos, imps, xerr=stds, color=colors, ecolor="grey", capsize=3, height=0.6)
|
||||
ax.set_yticks(y_pos)
|
||||
ax.set_yticklabels(names, fontsize=9)
|
||||
ax.invert_yaxis()
|
||||
ax.axvline(0, color="black", linewidth=0.8)
|
||||
ax.set_xlabel("Mean AUC drop (baseline − permuted)", fontsize=10)
|
||||
n_reps = len(set(r for r in range(len(all_results)))) # placeholder
|
||||
ax.set_title(
|
||||
f"MD Tower — Permutation Feature Importance\n"
|
||||
f"Phase 5 logit_mlp_head_ckpt ({len(all_results)} folds, "
|
||||
f"error bars = std across folds)",
|
||||
fontsize=11,
|
||||
)
|
||||
fig.tight_layout()
|
||||
out_png.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out_png, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {out_png}")
|
||||
|
||||
with open(out_csv, "w", newline="") as f:
|
||||
w = csv.DictWriter(f, fieldnames=["feature", "mean_importance", "std_importance"])
|
||||
w.writeheader()
|
||||
for feat in sorted_feats:
|
||||
w.writerow({"feature": feat,
|
||||
"mean_importance": agg[feat][0],
|
||||
"std_importance": agg[feat][1]})
|
||||
print(f"Saved: {out_csv}")
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _discover_checkpoints(ckpt_run: Path) -> list[tuple[int, int, Path]]:
|
||||
found = []
|
||||
for rep_dir in sorted(ckpt_run.glob("rep*")):
|
||||
try:
|
||||
rep_idx = int(rep_dir.name.replace("rep", ""))
|
||||
except ValueError:
|
||||
continue
|
||||
for fold_dir in sorted((rep_dir / "binary" / "ensemble").glob("fold[0-9]")):
|
||||
ckpt = fold_dir / "best_single.pt"
|
||||
if ckpt.exists():
|
||||
found.append((rep_idx, int(fold_dir.name.replace("fold", "")), ckpt))
|
||||
return found
|
||||
|
||||
|
||||
def main(n_permutations: int = N_PERMUTATIONS, seed: int = SEED):
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
print(f"Device: {device}")
|
||||
|
||||
print("Building DataBundle ...")
|
||||
data = build_data_bundle()
|
||||
print(f" feature_dim={data.feature_dim}")
|
||||
|
||||
checkpoints = _discover_checkpoints(CKPT_RUN)
|
||||
print(f"Found {len(checkpoints)} checkpoint(s) across "
|
||||
f"{len(set(r for r,f,_ in checkpoints))} rep(s)")
|
||||
|
||||
if not checkpoints:
|
||||
print("No checkpoints found.")
|
||||
return
|
||||
|
||||
all_results = []
|
||||
for rep_idx, fold_idx, ckpt in checkpoints:
|
||||
print(f"\n── rep{rep_idx:02d} fold{fold_idx} ──")
|
||||
model = build_model(ckpt, device)
|
||||
result = run_fold(rep_idx, fold_idx, model, data, device, n_permutations, seed)
|
||||
if result:
|
||||
all_results.append(result)
|
||||
del model
|
||||
|
||||
if not all_results:
|
||||
print("No results — nothing to plot.")
|
||||
return
|
||||
|
||||
plot_importance(
|
||||
all_results,
|
||||
FIGURES_ROOT / "md_importance_phase5.png",
|
||||
FIGURES_ROOT / "md_importance_phase5.csv",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--n-permutations", type=int, default=N_PERMUTATIONS)
|
||||
ap.add_argument("--seed", type=int, default=SEED)
|
||||
args = ap.parse_args()
|
||||
main(n_permutations=args.n_permutations, seed=args.seed)
|
||||
Reference in New Issue
Block a user