pre-refactor 041426
This commit is contained in:
rpotter6298
@@ -0,0 +1,264 @@
|
||||
"""
|
||||
Phase 5 comparison panel — ROC curves + fusion event summaries.
|
||||
|
||||
Layout:
|
||||
Top row (1 × 3) — ROC curves: Single HyperTower | Bilateral Ensemble | Fused Head
|
||||
Bottom rows (2 × 1) — Fusion event summary (full-width) for Ensemble then Fused Head
|
||||
(Single mode has fused-only bridge; no meaningful fusion events)
|
||||
|
||||
All data derived from predictions_test.csv — no checkpoints required.
|
||||
|
||||
Usage:
|
||||
python -m v3.scripts.output_analysis.explainability.comparison_panel_phase5
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.patches as mpatches
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.metrics import roc_auc_score, roc_curve
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
RESULTS_ROOT = REPO_ROOT / "v3" / "results"
|
||||
FIGURES_ROOT = REPO_ROOT / "v3" / "figures" / "explainability"
|
||||
CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
|
||||
|
||||
RUNS = [
|
||||
{"label": "Single HyperTower", "run": "phase5/single_fused",
|
||||
"tower_path": "binary/single", "is_single": False},
|
||||
{"label": "Bilateral Ensemble", "run": "phase5/ensemble_fused",
|
||||
"tower_path": "binary/ensemble", "is_single": False},
|
||||
{"label": "Fused Head", "run": "phase5/logit_mlp_head",
|
||||
"tower_path": "binary/ensemble", "is_single": False},
|
||||
]
|
||||
|
||||
# Event taxonomy (img=image tower, md=clinical tower)
|
||||
_EVENT_KEYS = [
|
||||
"full_correction", "img_assist", "md_assist",
|
||||
"full_error", "img_drag", "md_drag",
|
||||
"concordant_correct", "concordant_wrong",
|
||||
]
|
||||
_EVENT_COLORS = [
|
||||
"#2ca02c", "#98df8a", "#b5cf6b", # positive
|
||||
"#d62728", "#ff9896", "#ffbb78", # negative
|
||||
"#aec7e8", "#c5b0d5", # concordant
|
||||
]
|
||||
_POSITIVE_KEYS = _EVENT_KEYS[:3]
|
||||
_NEGATIVE_KEYS = _EVENT_KEYS[3:6]
|
||||
_DISAGREE_KEYS = _POSITIVE_KEYS + _NEGATIVE_KEYS # exclude concordant
|
||||
|
||||
|
||||
# ── Data loading ──────────────────────────────────────────────────────────────
|
||||
|
||||
def load_pooled(run: str, tower_path: str) -> pd.DataFrame:
|
||||
run_dir = RESULTS_ROOT / run
|
||||
rows = []
|
||||
for rep in sorted(run_dir.glob("rep*")):
|
||||
tm = rep / tower_path
|
||||
if not tm.exists():
|
||||
continue
|
||||
for fold in sorted(tm.glob("fold[0-9]")):
|
||||
csv = fold / "predictions_test.csv"
|
||||
if csv.exists():
|
||||
df = pd.read_csv(csv)
|
||||
df["rep"] = rep.name
|
||||
df["fold"] = fold.name
|
||||
rows.append(df)
|
||||
if not rows:
|
||||
raise FileNotFoundError(f"No predictions found under {run_dir}/{tower_path}")
|
||||
return pd.concat(rows, ignore_index=True)
|
||||
|
||||
|
||||
def classify_events(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Add event_type column based on pred_fused/pred_img/pred_md vs y_true."""
|
||||
df = df.copy()
|
||||
y = df["y_true"].values
|
||||
pf = df["pred_fused"].values
|
||||
pi = df["pred_img"].values
|
||||
pm = df["pred_md"].values
|
||||
|
||||
fused_ok = pf == y
|
||||
img_ok = pi == y
|
||||
md_ok = pm == y
|
||||
|
||||
def _classify(fo, io, mo):
|
||||
if fo and io and mo: return "concordant_correct"
|
||||
if not fo and not io and not mo: return "concordant_wrong"
|
||||
if fo and not io and not mo: return "full_correction"
|
||||
if fo and io and not mo: return "img_assist"
|
||||
if fo and not io and mo: return "md_assist"
|
||||
if not fo and io and mo: return "full_error"
|
||||
if not fo and not io and mo: return "img_drag"
|
||||
if not fo and io and not mo: return "md_drag"
|
||||
return "other"
|
||||
|
||||
df["event_type"] = [_classify(fo, io, mo)
|
||||
for fo, io, mo in zip(fused_ok, img_ok, md_ok)]
|
||||
# conf_delta: fused prob minus average of img/md
|
||||
df["conf_fused"] = df["prob_fused_c1"]
|
||||
df["conf_img"] = df["prob_img_c1"]
|
||||
df["conf_md"] = df["prob_md_c1"]
|
||||
df["conf_delta"] = df["conf_fused"] - 0.5 * (df["conf_img"] + df["conf_md"])
|
||||
return df
|
||||
|
||||
|
||||
# ── ROC panel ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def _draw_roc(ax, df: pd.DataFrame, label: str, color: str) -> None:
|
||||
"""Draw per-fold ROC curves (faint) + mean ROC (bold) on ax."""
|
||||
fold_aucs = []
|
||||
for (rep, fold), grp in df.groupby(["rep", "fold"]):
|
||||
if grp["y_true"].nunique() < 2:
|
||||
continue
|
||||
fpr, tpr, _ = roc_curve(grp["y_true"], grp["prob_fused_c1"])
|
||||
ax.plot(fpr, tpr, color=color, alpha=0.12, lw=0.8)
|
||||
fold_aucs.append(roc_auc_score(grp["y_true"], grp["prob_fused_c1"]))
|
||||
|
||||
# Mean ROC via interpolation
|
||||
mean_fpr = np.linspace(0, 1, 200)
|
||||
tprs = []
|
||||
for (rep, fold), grp in df.groupby(["rep", "fold"]):
|
||||
if grp["y_true"].nunique() < 2:
|
||||
continue
|
||||
fpr, tpr, _ = roc_curve(grp["y_true"], grp["prob_fused_c1"])
|
||||
tprs.append(np.interp(mean_fpr, fpr, tpr))
|
||||
mean_tpr = np.mean(tprs, axis=0)
|
||||
mean_auc = np.mean(fold_aucs)
|
||||
std_auc = np.std(fold_aucs)
|
||||
ax.plot(mean_fpr, mean_tpr, color=color, lw=2.2,
|
||||
label=f"Mean AUC = {mean_auc:.3f} ± {std_auc:.3f}")
|
||||
ax.fill_between(mean_fpr,
|
||||
np.percentile(tprs, 25, axis=0),
|
||||
np.percentile(tprs, 75, axis=0),
|
||||
color=color, alpha=0.12)
|
||||
ax.plot([0, 1], [0, 1], "k--", lw=0.7, alpha=0.5)
|
||||
ax.set_xlim(-0.02, 1.02); ax.set_ylim(-0.02, 1.02)
|
||||
ax.set_xlabel("False Positive Rate", fontsize=9)
|
||||
ax.set_ylabel("True Positive Rate", fontsize=9)
|
||||
ax.set_title(label, fontsize=10, fontweight="bold")
|
||||
ax.legend(fontsize=8, loc="lower right")
|
||||
ax.grid(alpha=0.25)
|
||||
|
||||
|
||||
# ── Fusion summary panel ──────────────────────────────────────────────────────
|
||||
|
||||
def _draw_fusion_summary(axes_row, df: pd.DataFrame, label: str) -> None:
|
||||
"""Draw 3-panel fusion summary (disagreement events only) on axes_row (list of 3 axes)."""
|
||||
event_color = dict(zip(_EVENT_KEYS, _EVENT_COLORS))
|
||||
event_labels = {
|
||||
"full_correction": "Full correction\n(both wrong → right)",
|
||||
"img_assist": "Img assist\n(img✓ md✗ → right)",
|
||||
"md_assist": "MD assist\n(md✓ img✗ → right)",
|
||||
"full_error": "Full error\n(both right → wrong)",
|
||||
"img_drag": "Img drag\n(img✗ md✓ → wrong)",
|
||||
"md_drag": "MD drag\n(md✗ img✓ → wrong)",
|
||||
}
|
||||
|
||||
# Only count disagreement events (exclude concordant)
|
||||
counts = {k: (df["event_type"] == k).sum() for k in _DISAGREE_KEYS}
|
||||
|
||||
# Panel 0: totals bar (positive vs negative)
|
||||
ax = axes_row[0]
|
||||
for bar_x, keys in ((0, _POSITIVE_KEYS), (1, _NEGATIVE_KEYS)):
|
||||
bot = 0
|
||||
for k in keys:
|
||||
c = int(counts[k])
|
||||
ax.bar(bar_x, c, bottom=bot, color=event_color[k], width=0.5)
|
||||
if c > 0:
|
||||
ax.text(bar_x, bot + c / 2, str(c), ha="center", va="center",
|
||||
fontsize=8, fontweight="bold")
|
||||
bot += c
|
||||
ax.set_xticks([0, 1]); ax.set_xticklabels(["Positive\nevents", "Negative\nevents"])
|
||||
ax.set_ylabel("Count (all folds)")
|
||||
patches = [mpatches.Patch(color=event_color[k], label=event_labels[k].split("\n")[0])
|
||||
for k in _DISAGREE_KEYS if counts[k] > 0]
|
||||
ax.legend(handles=patches, fontsize=6, loc="upper right")
|
||||
ax.set_title(f"{label}\nDisagreement event totals", fontsize=9)
|
||||
|
||||
# Panel 1: per-fold stacked bar (disagreement events only)
|
||||
ax = axes_row[1]
|
||||
fold_groups = sorted(df.groupby(["rep", "fold"]), key=lambda x: x[0])
|
||||
x = np.arange(len(fold_groups))
|
||||
pos_bot = np.zeros(len(fold_groups))
|
||||
neg_bot = np.zeros(len(fold_groups))
|
||||
for k, color in zip(_POSITIVE_KEYS, _EVENT_COLORS[:3]):
|
||||
vals = np.array([(g["event_type"] == k).sum() for _, g in fold_groups], dtype=float)
|
||||
ax.bar(x, vals, bottom=pos_bot, color=color, width=0.6)
|
||||
pos_bot += vals
|
||||
for k, color in zip(_NEGATIVE_KEYS, _EVENT_COLORS[3:6]):
|
||||
vals = np.array([(g["event_type"] == k).sum() for _, g in fold_groups], dtype=float)
|
||||
ax.bar(x + 0.65, vals, bottom=neg_bot, color=color, width=0.6)
|
||||
neg_bot += vals
|
||||
ax.set_xticks([])
|
||||
ax.set_xlabel("Fold", fontsize=8)
|
||||
ax.set_ylabel("Count"); ax.set_title("Per-fold breakdown\n(left=positive, right=negative)", fontsize=9)
|
||||
|
||||
# Panel 2: img vs md confidence scatter (disagreement events only)
|
||||
ax = axes_row[2]
|
||||
for k in _DISAGREE_KEYS:
|
||||
sub = df[df["event_type"] == k]
|
||||
if len(sub) == 0:
|
||||
continue
|
||||
ax.scatter(sub["conf_img"], sub["conf_md"], c=event_color[k],
|
||||
alpha=0.65, s=30, edgecolors="none",
|
||||
label=event_labels[k].split("\n")[0])
|
||||
ax.plot([0, 1], [0, 1], "k--", lw=0.5, alpha=0.4)
|
||||
ax.set_xlabel("P(Glaucoma) — Image head"); ax.set_ylabel("P(Glaucoma) — MD head")
|
||||
ax.legend(fontsize=5.5, loc="lower right")
|
||||
ax.set_title("Tower confidence space\n(disagreement events only)", fontsize=9)
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
ROC_COLORS = ["#4c72b0", "#dd8452", "#55a868"]
|
||||
|
||||
print("Loading predictions ...")
|
||||
datasets = []
|
||||
for cfg, color in zip(RUNS, ROC_COLORS):
|
||||
df = load_pooled(cfg["run"], cfg["tower_path"])
|
||||
df = classify_events(df)
|
||||
datasets.append((cfg, df, color))
|
||||
|
||||
fusion_runs = [(cfg, df, color) for cfg, df, color in datasets]
|
||||
|
||||
# ── Layout ────────────────────────────────────────────────────────────────
|
||||
# Row 0: 3 ROC axes
|
||||
# Rows 1,2: 2 fusion summary strips (5 axes each, spanning full width)
|
||||
n_fusion = len(fusion_runs)
|
||||
fig = plt.figure(figsize=(20, 6 + 4.5 * n_fusion))
|
||||
gs = fig.add_gridspec(
|
||||
1 + n_fusion, 1,
|
||||
height_ratios=[5] + [4.5] * n_fusion,
|
||||
hspace=0.35,
|
||||
)
|
||||
|
||||
# ROC row — subdivide into 3
|
||||
roc_gs = gs[0].subgridspec(1, 3, wspace=0.28)
|
||||
for i, (cfg, df, color) in enumerate(datasets):
|
||||
ax = fig.add_subplot(roc_gs[i])
|
||||
_draw_roc(ax, df, cfg["label"], color)
|
||||
|
||||
# Fusion rows (3 panels each)
|
||||
for fi, (cfg, df, color) in enumerate(fusion_runs):
|
||||
fus_gs = gs[1 + fi].subgridspec(1, 3, wspace=0.32)
|
||||
axes_row = [fig.add_subplot(fus_gs[j]) for j in range(3)]
|
||||
_draw_fusion_summary(axes_row, df, cfg["label"])
|
||||
|
||||
fig.suptitle("Phase 5 — Model Comparison: ROC Curves & Fusion Event Analysis",
|
||||
fontsize=13, fontweight="bold", y=1.01)
|
||||
|
||||
out = FIGURES_ROOT / "comparison_panel_phase5.png"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user