rpotter6298
265 lines
11 KiB
Python
265 lines
11 KiB
Python
"""
|
||
Phase 5 comparison panel — ROC curves + fusion event summaries.
|
||
|
||
Layout:
|
||
Top row (1 × 3) — ROC curves: Single HyperTower | Bilateral Ensemble | Fused Head
|
||
Bottom rows (2 × 1) — Fusion event summary (full-width) for Ensemble then Fused Head
|
||
(Single mode has fused-only bridge; no meaningful fusion events)
|
||
|
||
All data derived from predictions_test.csv — no checkpoints required.
|
||
|
||
Usage:
|
||
python -m v3.scripts.output_analysis.explainability.comparison_panel_phase5
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from pathlib import Path
|
||
|
||
import matplotlib
|
||
matplotlib.use("Agg")
|
||
import matplotlib.patches as mpatches
|
||
import matplotlib.pyplot as plt
|
||
import numpy as np
|
||
import pandas as pd
|
||
from sklearn.metrics import roc_auc_score, roc_curve
|
||
|
||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||
RESULTS_ROOT = REPO_ROOT / "v3" / "results"
|
||
FIGURES_ROOT = REPO_ROOT / "v3" / "figures" / "explainability"
|
||
CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
|
||
|
||
RUNS = [
|
||
{"label": "Single HyperTower", "run": "phase5/single_fused",
|
||
"tower_path": "binary/single", "is_single": False},
|
||
{"label": "Bilateral Ensemble", "run": "phase5/ensemble_fused",
|
||
"tower_path": "binary/ensemble", "is_single": False},
|
||
{"label": "Fused Head", "run": "phase5/logit_mlp_head",
|
||
"tower_path": "binary/ensemble", "is_single": False},
|
||
]
|
||
|
||
# Event taxonomy (img=image tower, md=clinical tower)
|
||
_EVENT_KEYS = [
|
||
"full_correction", "img_assist", "md_assist",
|
||
"full_error", "img_drag", "md_drag",
|
||
"concordant_correct", "concordant_wrong",
|
||
]
|
||
_EVENT_COLORS = [
|
||
"#2ca02c", "#98df8a", "#b5cf6b", # positive
|
||
"#d62728", "#ff9896", "#ffbb78", # negative
|
||
"#aec7e8", "#c5b0d5", # concordant
|
||
]
|
||
_POSITIVE_KEYS = _EVENT_KEYS[:3]
|
||
_NEGATIVE_KEYS = _EVENT_KEYS[3:6]
|
||
_DISAGREE_KEYS = _POSITIVE_KEYS + _NEGATIVE_KEYS # exclude concordant
|
||
|
||
|
||
# ── Data loading ──────────────────────────────────────────────────────────────
|
||
|
||
def load_pooled(run: str, tower_path: str) -> pd.DataFrame:
|
||
run_dir = RESULTS_ROOT / run
|
||
rows = []
|
||
for rep in sorted(run_dir.glob("rep*")):
|
||
tm = rep / tower_path
|
||
if not tm.exists():
|
||
continue
|
||
for fold in sorted(tm.glob("fold[0-9]")):
|
||
csv = fold / "predictions_test.csv"
|
||
if csv.exists():
|
||
df = pd.read_csv(csv)
|
||
df["rep"] = rep.name
|
||
df["fold"] = fold.name
|
||
rows.append(df)
|
||
if not rows:
|
||
raise FileNotFoundError(f"No predictions found under {run_dir}/{tower_path}")
|
||
return pd.concat(rows, ignore_index=True)
|
||
|
||
|
||
def classify_events(df: pd.DataFrame) -> pd.DataFrame:
|
||
"""Add event_type column based on pred_fused/pred_img/pred_md vs y_true."""
|
||
df = df.copy()
|
||
y = df["y_true"].values
|
||
pf = df["pred_fused"].values
|
||
pi = df["pred_img"].values
|
||
pm = df["pred_md"].values
|
||
|
||
fused_ok = pf == y
|
||
img_ok = pi == y
|
||
md_ok = pm == y
|
||
|
||
def _classify(fo, io, mo):
|
||
if fo and io and mo: return "concordant_correct"
|
||
if not fo and not io and not mo: return "concordant_wrong"
|
||
if fo and not io and not mo: return "full_correction"
|
||
if fo and io and not mo: return "img_assist"
|
||
if fo and not io and mo: return "md_assist"
|
||
if not fo and io and mo: return "full_error"
|
||
if not fo and not io and mo: return "img_drag"
|
||
if not fo and io and not mo: return "md_drag"
|
||
return "other"
|
||
|
||
df["event_type"] = [_classify(fo, io, mo)
|
||
for fo, io, mo in zip(fused_ok, img_ok, md_ok)]
|
||
# conf_delta: fused prob minus average of img/md
|
||
df["conf_fused"] = df["prob_fused_c1"]
|
||
df["conf_img"] = df["prob_img_c1"]
|
||
df["conf_md"] = df["prob_md_c1"]
|
||
df["conf_delta"] = df["conf_fused"] - 0.5 * (df["conf_img"] + df["conf_md"])
|
||
return df
|
||
|
||
|
||
# ── ROC panel ─────────────────────────────────────────────────────────────────
|
||
|
||
def _draw_roc(ax, df: pd.DataFrame, label: str, color: str) -> None:
|
||
"""Draw per-fold ROC curves (faint) + mean ROC (bold) on ax."""
|
||
fold_aucs = []
|
||
for (rep, fold), grp in df.groupby(["rep", "fold"]):
|
||
if grp["y_true"].nunique() < 2:
|
||
continue
|
||
fpr, tpr, _ = roc_curve(grp["y_true"], grp["prob_fused_c1"])
|
||
ax.plot(fpr, tpr, color=color, alpha=0.12, lw=0.8)
|
||
fold_aucs.append(roc_auc_score(grp["y_true"], grp["prob_fused_c1"]))
|
||
|
||
# Mean ROC via interpolation
|
||
mean_fpr = np.linspace(0, 1, 200)
|
||
tprs = []
|
||
for (rep, fold), grp in df.groupby(["rep", "fold"]):
|
||
if grp["y_true"].nunique() < 2:
|
||
continue
|
||
fpr, tpr, _ = roc_curve(grp["y_true"], grp["prob_fused_c1"])
|
||
tprs.append(np.interp(mean_fpr, fpr, tpr))
|
||
mean_tpr = np.mean(tprs, axis=0)
|
||
mean_auc = np.mean(fold_aucs)
|
||
std_auc = np.std(fold_aucs)
|
||
ax.plot(mean_fpr, mean_tpr, color=color, lw=2.2,
|
||
label=f"Mean AUC = {mean_auc:.3f} ± {std_auc:.3f}")
|
||
ax.fill_between(mean_fpr,
|
||
np.percentile(tprs, 25, axis=0),
|
||
np.percentile(tprs, 75, axis=0),
|
||
color=color, alpha=0.12)
|
||
ax.plot([0, 1], [0, 1], "k--", lw=0.7, alpha=0.5)
|
||
ax.set_xlim(-0.02, 1.02); ax.set_ylim(-0.02, 1.02)
|
||
ax.set_xlabel("False Positive Rate", fontsize=9)
|
||
ax.set_ylabel("True Positive Rate", fontsize=9)
|
||
ax.set_title(label, fontsize=10, fontweight="bold")
|
||
ax.legend(fontsize=8, loc="lower right")
|
||
ax.grid(alpha=0.25)
|
||
|
||
|
||
# ── Fusion summary panel ──────────────────────────────────────────────────────
|
||
|
||
def _draw_fusion_summary(axes_row, df: pd.DataFrame, label: str) -> None:
|
||
"""Draw 3-panel fusion summary (disagreement events only) on axes_row (list of 3 axes)."""
|
||
event_color = dict(zip(_EVENT_KEYS, _EVENT_COLORS))
|
||
event_labels = {
|
||
"full_correction": "Full correction\n(both wrong → right)",
|
||
"img_assist": "Img assist\n(img✓ md✗ → right)",
|
||
"md_assist": "MD assist\n(md✓ img✗ → right)",
|
||
"full_error": "Full error\n(both right → wrong)",
|
||
"img_drag": "Img drag\n(img✗ md✓ → wrong)",
|
||
"md_drag": "MD drag\n(md✗ img✓ → wrong)",
|
||
}
|
||
|
||
# Only count disagreement events (exclude concordant)
|
||
counts = {k: (df["event_type"] == k).sum() for k in _DISAGREE_KEYS}
|
||
|
||
# Panel 0: totals bar (positive vs negative)
|
||
ax = axes_row[0]
|
||
for bar_x, keys in ((0, _POSITIVE_KEYS), (1, _NEGATIVE_KEYS)):
|
||
bot = 0
|
||
for k in keys:
|
||
c = int(counts[k])
|
||
ax.bar(bar_x, c, bottom=bot, color=event_color[k], width=0.5)
|
||
if c > 0:
|
||
ax.text(bar_x, bot + c / 2, str(c), ha="center", va="center",
|
||
fontsize=8, fontweight="bold")
|
||
bot += c
|
||
ax.set_xticks([0, 1]); ax.set_xticklabels(["Positive\nevents", "Negative\nevents"])
|
||
ax.set_ylabel("Count (all folds)")
|
||
patches = [mpatches.Patch(color=event_color[k], label=event_labels[k].split("\n")[0])
|
||
for k in _DISAGREE_KEYS if counts[k] > 0]
|
||
ax.legend(handles=patches, fontsize=6, loc="upper right")
|
||
ax.set_title(f"{label}\nDisagreement event totals", fontsize=9)
|
||
|
||
# Panel 1: per-fold stacked bar (disagreement events only)
|
||
ax = axes_row[1]
|
||
fold_groups = sorted(df.groupby(["rep", "fold"]), key=lambda x: x[0])
|
||
x = np.arange(len(fold_groups))
|
||
pos_bot = np.zeros(len(fold_groups))
|
||
neg_bot = np.zeros(len(fold_groups))
|
||
for k, color in zip(_POSITIVE_KEYS, _EVENT_COLORS[:3]):
|
||
vals = np.array([(g["event_type"] == k).sum() for _, g in fold_groups], dtype=float)
|
||
ax.bar(x, vals, bottom=pos_bot, color=color, width=0.6)
|
||
pos_bot += vals
|
||
for k, color in zip(_NEGATIVE_KEYS, _EVENT_COLORS[3:6]):
|
||
vals = np.array([(g["event_type"] == k).sum() for _, g in fold_groups], dtype=float)
|
||
ax.bar(x + 0.65, vals, bottom=neg_bot, color=color, width=0.6)
|
||
neg_bot += vals
|
||
ax.set_xticks([])
|
||
ax.set_xlabel("Fold", fontsize=8)
|
||
ax.set_ylabel("Count"); ax.set_title("Per-fold breakdown\n(left=positive, right=negative)", fontsize=9)
|
||
|
||
# Panel 2: img vs md confidence scatter (disagreement events only)
|
||
ax = axes_row[2]
|
||
for k in _DISAGREE_KEYS:
|
||
sub = df[df["event_type"] == k]
|
||
if len(sub) == 0:
|
||
continue
|
||
ax.scatter(sub["conf_img"], sub["conf_md"], c=event_color[k],
|
||
alpha=0.65, s=30, edgecolors="none",
|
||
label=event_labels[k].split("\n")[0])
|
||
ax.plot([0, 1], [0, 1], "k--", lw=0.5, alpha=0.4)
|
||
ax.set_xlabel("P(Glaucoma) — Image head"); ax.set_ylabel("P(Glaucoma) — MD head")
|
||
ax.legend(fontsize=5.5, loc="lower right")
|
||
ax.set_title("Tower confidence space\n(disagreement events only)", fontsize=9)
|
||
|
||
|
||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||
|
||
def main():
|
||
ROC_COLORS = ["#4c72b0", "#dd8452", "#55a868"]
|
||
|
||
print("Loading predictions ...")
|
||
datasets = []
|
||
for cfg, color in zip(RUNS, ROC_COLORS):
|
||
df = load_pooled(cfg["run"], cfg["tower_path"])
|
||
df = classify_events(df)
|
||
datasets.append((cfg, df, color))
|
||
|
||
fusion_runs = [(cfg, df, color) for cfg, df, color in datasets]
|
||
|
||
# ── Layout ────────────────────────────────────────────────────────────────
|
||
# Row 0: 3 ROC axes
|
||
# Rows 1,2: 2 fusion summary strips (5 axes each, spanning full width)
|
||
n_fusion = len(fusion_runs)
|
||
fig = plt.figure(figsize=(20, 6 + 4.5 * n_fusion))
|
||
gs = fig.add_gridspec(
|
||
1 + n_fusion, 1,
|
||
height_ratios=[5] + [4.5] * n_fusion,
|
||
hspace=0.35,
|
||
)
|
||
|
||
# ROC row — subdivide into 3
|
||
roc_gs = gs[0].subgridspec(1, 3, wspace=0.28)
|
||
for i, (cfg, df, color) in enumerate(datasets):
|
||
ax = fig.add_subplot(roc_gs[i])
|
||
_draw_roc(ax, df, cfg["label"], color)
|
||
|
||
# Fusion rows (3 panels each)
|
||
for fi, (cfg, df, color) in enumerate(fusion_runs):
|
||
fus_gs = gs[1 + fi].subgridspec(1, 3, wspace=0.32)
|
||
axes_row = [fig.add_subplot(fus_gs[j]) for j in range(3)]
|
||
_draw_fusion_summary(axes_row, df, cfg["label"])
|
||
|
||
fig.suptitle("Phase 5 — Model Comparison: ROC Curves & Fusion Event Analysis",
|
||
fontsize=13, fontweight="bold", y=1.01)
|
||
|
||
out = FIGURES_ROOT / "comparison_panel_phase5.png"
|
||
out.parent.mkdir(parents=True, exist_ok=True)
|
||
fig.savefig(out, dpi=150, bbox_inches="tight")
|
||
plt.close(fig)
|
||
print(f"Saved: {out}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|