Add new experiments and analysis scripts for dropzero features
- Introduced `fused_importance_with_axial.py` to evaluate the importance of Axial_Length in the fused-head model. - Created JSON configurations for various experiments excluding zero-importance clinical features: - `cd_solo_bilateral_dropzero.json`: Bilateral clinical-only evaluation. - `cd_solo_single_dropzero.json`: Single-eye clinical-only evaluation. - `ensemble_refugelike_ckpt_dropzero.json`: Ensemble model with dropped zero-importance features. - `ensemble_single_refugelike_dropzero.json`: Single-eye ensemble model with dropped features.
This commit is contained in:
rpotter6298
+322
-10
@@ -18,7 +18,7 @@ from pathlib import Path
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.patches import FancyBboxPatch
|
||||
from matplotlib.patches import FancyBboxPatch, Rectangle
|
||||
|
||||
OUT = Path(__file__).parent / "output" / "F1_architecture.png"
|
||||
|
||||
@@ -76,6 +76,15 @@ def _bracket(ax, x, y0, y1, text="", pad=0.20, fontsize=9, badge_color="#555"):
|
||||
boxstyle="round,pad=0.3"))
|
||||
|
||||
|
||||
def _tag(ax, cx, cy, letter, color="#222"):
|
||||
"""Small square tag in the top-right corner of a main-diagram box."""
|
||||
ax.text(cx, cy, letter, ha="center", va="center",
|
||||
fontsize=8.5, color="white", fontfamily=FONT, fontweight="bold",
|
||||
zorder=6,
|
||||
bbox=dict(facecolor=color, edgecolor="white", linewidth=1.0,
|
||||
boxstyle="round,pad=0.20"))
|
||||
|
||||
|
||||
def _draw_output(ax, x, y, classes=("Glaucoma", "Normal")):
|
||||
bw, bh, gap = 1.10, 0.38, 0.08
|
||||
n = len(classes)
|
||||
@@ -100,10 +109,12 @@ def _draw_eye_fusion(ax, x_left, y_img, y_md, eye_label):
|
||||
|
||||
# Image network box
|
||||
_box(ax, x_left + bw_img / 2, y_img, bw_img, bh_img, C_IMG,
|
||||
f"{eye_label}\nImage Network", fontsize=8.5, radius=0.08)
|
||||
f"{eye_label}\nCNN", fontsize=8.5, radius=0.08)
|
||||
_tag(ax, x_left + bw_img - 0.10, y_img + bh_img / 2 - 0.10, "A", color=C_IMG)
|
||||
# Clinical network box
|
||||
_box(ax, x_left + bw_md / 2, y_md, bw_md, bh_md, C_MD,
|
||||
f"{eye_label}\nClinical Network", fontsize=8.5, radius=0.08)
|
||||
f"{eye_label}\nMLP", fontsize=8.5, radius=0.08)
|
||||
_tag(ax, x_left + bw_md - 0.10, y_md + bh_md / 2 - 0.10, "B", color=C_MD)
|
||||
|
||||
# Fusion bridge — with math detail (replaces compact "Bridge" label)
|
||||
br_x = x_left + max(bw_img, bw_md) + 1.40
|
||||
@@ -111,18 +122,292 @@ def _draw_eye_fusion(ax, x_left, y_img, y_md, eye_label):
|
||||
_arrow(ax, x_left + bw_img, y_img, br_x - bw_br / 2, cy_br, lw=1.2, style="-|>")
|
||||
_arrow(ax, x_left + bw_md, y_md, br_x - bw_br / 2, cy_br, lw=1.2, style="-|>")
|
||||
_box(ax, br_x, cy_br, bw_br, bh_br, C_BRIDGE,
|
||||
"Fusion Bridge\nFC(img → 256)\nFC(md → 256)\nHadamard product",
|
||||
fontsize=8, radius=0.10)
|
||||
"Eye-Level Fusion\n(Hadamard Product)",
|
||||
fontsize=8.5, radius=0.10)
|
||||
_tag(ax, br_x + bw_br / 2 - 0.12, cy_br + bh_br / 2 - 0.12, "C", color=C_BRIDGE)
|
||||
return br_x + bw_br / 2, cy_br
|
||||
|
||||
|
||||
# ── Detail-row primitives ────────────────────────────────────────────────────
|
||||
|
||||
def _rect(ax, cx, cy, w, h, color, alpha=0.92, lw=0.0, edge="none"):
|
||||
ax.add_patch(Rectangle((cx - w / 2, cy - h / 2), w, h,
|
||||
facecolor=color, edgecolor=edge, linewidth=lw,
|
||||
alpha=alpha, zorder=3))
|
||||
|
||||
|
||||
def _vector(ax, cx, cy, n_cells, cell_h=0.14, cell_w=0.30, color=C_INPUT,
|
||||
alpha=0.92, gap=0.02):
|
||||
"""Draw a vertical vector of n_cells stacked cells centred at (cx, cy)."""
|
||||
total = n_cells * cell_h + (n_cells - 1) * gap
|
||||
y_top = cy + total / 2 - cell_h / 2
|
||||
for i in range(n_cells):
|
||||
y = y_top - i * (cell_h + gap)
|
||||
_rect(ax, cx, y, cell_w, cell_h, color, alpha=alpha, lw=0.6, edge="white")
|
||||
|
||||
|
||||
def _setup_detail_panel(ax, xlim=(0, 5), ylim=(0, 5)):
|
||||
ax.set_xlim(*xlim); ax.set_ylim(*ylim)
|
||||
ax.set_xticks([]); ax.set_yticks([])
|
||||
ax.set_facecolor(C_BG)
|
||||
for spine in ax.spines.values():
|
||||
spine.set_color("#999")
|
||||
spine.set_linewidth(0.6)
|
||||
|
||||
|
||||
def _draw_cnn_detail(ax):
|
||||
"""CNN schematic: Image -> shrinking feature maps -> pooled vector."""
|
||||
_setup_detail_panel(ax)
|
||||
_text(ax, 2.5, 4.65, "CNN", fontsize=10.5, color="#222", bold=True)
|
||||
_tag(ax, 0.30, 4.70, "A", color=C_IMG)
|
||||
|
||||
# Image
|
||||
_rect(ax, 0.55, 2.5, 0.80, 1.10, C_INPUT, alpha=0.85, lw=0.6, edge="white")
|
||||
_text(ax, 0.55, 2.5, "Fundus\nImage", fontsize=7.0, color="#333")
|
||||
# Feature maps: shrinking spatially, stacked in a row
|
||||
map_xs = [1.55, 2.20, 2.75, 3.20]
|
||||
map_hs = [1.00, 0.85, 0.70, 0.55]
|
||||
for x, h in zip(map_xs, map_hs):
|
||||
_rect(ax, x, 2.5, h * 0.60, h, C_IMG, alpha=0.75, lw=0.6, edge="white")
|
||||
# Pooled vector
|
||||
_vector(ax, 3.90, 2.50, 8, cell_h=0.13, cell_w=0.30, color=C_IMG, alpha=0.92)
|
||||
# Arrows
|
||||
prev_x = 0.55 + 0.40
|
||||
for x, h in zip(map_xs, map_hs):
|
||||
_arrow(ax, prev_x, 2.50, x - h * 0.30, 2.50, lw=0.8)
|
||||
prev_x = x + h * 0.30
|
||||
_arrow(ax, prev_x, 2.50, 3.90 - 0.15, 2.50, lw=0.8)
|
||||
|
||||
_text(ax, 3.90, 1.05, "z_img\n(2048-d)", fontsize=7.5, color="#333")
|
||||
_text(ax, 2.50, 0.35,
|
||||
"Conv + pool → global avg pool → flatten",
|
||||
fontsize=7.8, color="#555")
|
||||
|
||||
|
||||
def _draw_mlp_detail(ax):
|
||||
"""MLP schematic: input features -> Linear+LN+ReLU -> Linear+ReLU -> embedding.
|
||||
|
||||
Two Linear layers total (matching ClinicalEncoder's block0 + block1). The
|
||||
single circle column represents the hidden 128-d activations after the
|
||||
first Linear + LayerNorm + ReLU + Dropout; the output vector represents
|
||||
the 128-d embedding after the second Linear + ReLU.
|
||||
"""
|
||||
_setup_detail_panel(ax)
|
||||
_text(ax, 2.5, 4.65, "MLP", fontsize=10.5, color="#222", bold=True)
|
||||
_tag(ax, 0.30, 4.70, "B", color=C_MD)
|
||||
|
||||
labels = ["Age", "IOP", "Gender", "Pachy.", "..."]
|
||||
n_in = len(labels)
|
||||
y_top_in = 2.50 + (n_in * 0.28) / 2 - 0.14
|
||||
for i, lbl in enumerate(labels):
|
||||
y = y_top_in - i * 0.28
|
||||
_rect(ax, 0.55, y, 0.75, 0.22, C_MD, alpha=0.85, lw=0.6, edge="white")
|
||||
_text(ax, 0.55, y, lbl, fontsize=6.8, color="white")
|
||||
|
||||
# Single hidden column (post-Linear + LN + ReLU + Dropout)
|
||||
cx_hidden, n_hidden = 2.15, 5
|
||||
y_top_h = 2.50 + (n_hidden * 0.32) / 2 - 0.16
|
||||
circle_ys = [y_top_h - j * 0.32 for j in range(n_hidden)]
|
||||
for y in circle_ys:
|
||||
ax.add_patch(plt.Circle((cx_hidden, y), 0.10, facecolor=C_MD,
|
||||
edgecolor="white", lw=0.6, alpha=0.9, zorder=3))
|
||||
|
||||
# LN badge sitting above the hidden column (annotates LN applied at this stage)
|
||||
_box(ax, cx_hidden, 3.55, 0.36, 0.28, C_MD, "LN",
|
||||
fontsize=6.5, radius=0.05, alpha=0.75)
|
||||
|
||||
# Output vector (post-Linear + ReLU)
|
||||
_vector(ax, 3.55, 2.50, 6, cell_h=0.13, cell_w=0.28, color=C_MD, alpha=0.92)
|
||||
|
||||
# Dense connections: input → hidden (Linear 1)
|
||||
for i in range(n_in):
|
||||
y0 = y_top_in - i * 0.28
|
||||
for y1 in circle_ys:
|
||||
ax.plot([0.95, cx_hidden - 0.10], [y0, y1],
|
||||
color="#888", lw=0.25, alpha=0.4, zorder=2)
|
||||
# Dense connections: hidden → output vector (Linear 2)
|
||||
out_cell_ys = [2.50 + (6 * 0.15) / 2 - 0.075 - k * 0.15 for k in range(6)]
|
||||
for y0 in circle_ys:
|
||||
for y1 in out_cell_ys:
|
||||
ax.plot([cx_hidden + 0.10, 3.55 - 0.14], [y0, y1],
|
||||
color="#888", lw=0.25, alpha=0.4, zorder=2)
|
||||
|
||||
_text(ax, 3.55, 1.05, "z_cd\n(128-d)", fontsize=7.5, color="#333")
|
||||
_text(ax, 2.50, 0.30,
|
||||
"Linear + LN + ReLU + Drop\n→ Linear + ReLU",
|
||||
fontsize=7.5, color="#555")
|
||||
|
||||
|
||||
def _draw_eye_fusion_detail(ax):
|
||||
"""Eye-level fusion: each modality Linear-projected to shared dim, Hadamard."""
|
||||
_setup_detail_panel(ax)
|
||||
_text(ax, 2.5, 4.65, "Eye-Level Fusion", fontsize=10.5, color="#222", bold=True)
|
||||
_tag(ax, 0.30, 4.70, "C", color=C_BRIDGE)
|
||||
|
||||
# ── Top row: image path (z_img → dense lines → W_img circles → LN → ⊙)
|
||||
cell_h_img, cell_w_img = 0.08, 0.20
|
||||
n_img = 12
|
||||
y_img_ctr = 3.45
|
||||
_vector(ax, 0.40, y_img_ctr, n_img, cell_h=cell_h_img, cell_w=cell_w_img,
|
||||
color=C_IMG)
|
||||
_text(ax, 0.40, 4.20, "z_img", fontsize=6.6, color="#333")
|
||||
|
||||
cx_wimg, n_wimg = 1.70, 6
|
||||
y_top_wimg = y_img_ctr + (n_wimg * 0.17) / 2 - 0.085
|
||||
circle_ys_wimg = [y_top_wimg - j * 0.17 for j in range(n_wimg)]
|
||||
for y in circle_ys_wimg:
|
||||
ax.add_patch(plt.Circle((cx_wimg, y), 0.075, facecolor=C_IMG,
|
||||
edgecolor="white", lw=0.5, alpha=0.9, zorder=3))
|
||||
_text(ax, cx_wimg, 4.20, "W_img 2048→256", fontsize=6.2, color="#555")
|
||||
|
||||
# Dense connections z_img cells → W_img circles
|
||||
cell_step_img = cell_h_img + 0.02
|
||||
img_cell_ys = [y_img_ctr + (n_img * cell_h_img + (n_img - 1) * 0.02) / 2
|
||||
- cell_h_img / 2 - j * cell_step_img for j in range(n_img)]
|
||||
for y_src in img_cell_ys:
|
||||
for y_dst in circle_ys_wimg:
|
||||
ax.plot([0.40 + cell_w_img / 2, cx_wimg - 0.075],
|
||||
[y_src, y_dst], color="#888", lw=0.2, alpha=0.25, zorder=1)
|
||||
|
||||
_arrow(ax, cx_wimg + 0.10, y_img_ctr, 2.15, y_img_ctr, lw=0.8)
|
||||
_box(ax, 2.35, y_img_ctr, 0.30, 0.30, C_IMG, "LN",
|
||||
fontsize=6.5, radius=0.05, alpha=0.75)
|
||||
|
||||
# ── Bottom row: clinical path (z_cd → dense lines → W_cd circles → LN → ⊙)
|
||||
cell_h_cd, cell_w_cd = 0.15, 0.20
|
||||
n_cd = 4
|
||||
y_cd_ctr = 1.55
|
||||
_vector(ax, 0.40, y_cd_ctr, n_cd, cell_h=cell_h_cd, cell_w=cell_w_cd,
|
||||
color=C_MD)
|
||||
_text(ax, 0.40, 0.85, "z_cd", fontsize=6.6, color="#333")
|
||||
|
||||
cx_wcd, n_wcd = 1.70, 6
|
||||
y_top_wcd = y_cd_ctr + (n_wcd * 0.17) / 2 - 0.085
|
||||
circle_ys_wcd = [y_top_wcd - j * 0.17 for j in range(n_wcd)]
|
||||
for y in circle_ys_wcd:
|
||||
ax.add_patch(plt.Circle((cx_wcd, y), 0.075, facecolor=C_MD,
|
||||
edgecolor="white", lw=0.5, alpha=0.9, zorder=3))
|
||||
_text(ax, cx_wcd, 0.85, "W_cd 128→256", fontsize=6.2, color="#555")
|
||||
|
||||
cell_step_cd = cell_h_cd + 0.02
|
||||
cd_cell_ys = [y_cd_ctr + (n_cd * cell_h_cd + (n_cd - 1) * 0.02) / 2
|
||||
- cell_h_cd / 2 - j * cell_step_cd for j in range(n_cd)]
|
||||
for y_src in cd_cell_ys:
|
||||
for y_dst in circle_ys_wcd:
|
||||
ax.plot([0.40 + cell_w_cd / 2, cx_wcd - 0.075],
|
||||
[y_src, y_dst], color="#888", lw=0.2, alpha=0.25, zorder=1)
|
||||
|
||||
_arrow(ax, cx_wcd + 0.10, y_cd_ctr, 2.15, y_cd_ctr, lw=0.8)
|
||||
_box(ax, 2.35, y_cd_ctr, 0.30, 0.30, C_MD, "LN",
|
||||
fontsize=6.5, radius=0.05, alpha=0.75)
|
||||
|
||||
# ── Convergence at ⊙ → z_fused
|
||||
_arrow(ax, 2.55, y_img_ctr, 3.10, 2.50, lw=0.9)
|
||||
_arrow(ax, 2.55, y_cd_ctr, 3.10, 2.50, lw=0.9)
|
||||
_text(ax, 3.35, 2.50, "⊙", fontsize=20, color="#333")
|
||||
_arrow(ax, 3.60, 2.50, 3.90, 2.50, lw=1.0)
|
||||
_vector(ax, 4.15, 2.50, 6, cell_h=0.12, cell_w=0.22, color=C_BRIDGE)
|
||||
_text(ax, 4.15, 1.65, "z_fused\n(256)", fontsize=6.8, color="#333")
|
||||
|
||||
_text(ax, 2.50, 0.30,
|
||||
"Project + LN each modality → Hadamard product",
|
||||
fontsize=7.8, color="#555")
|
||||
|
||||
|
||||
def _draw_patient_fusion_detail(ax):
|
||||
"""Concatenation schematic: two eye embeddings → concat → head → logits.
|
||||
|
||||
Draws z_OD and z_OS as vertically-stacked rectangles pushed close together
|
||||
with a right-side concat bracket, then two dense layers as columns of
|
||||
circles (matching MLP style): FC 512→256 (hb bridge) and FC 256→2 (hb head).
|
||||
"""
|
||||
_setup_detail_panel(ax)
|
||||
_text(ax, 2.5, 4.65, "Patient-Level Fusion", fontsize=10.5, color="#222", bold=True)
|
||||
_tag(ax, 0.30, 4.70, "D", color=C_HEAD)
|
||||
|
||||
# z_OD and z_OS: vertically stacked rectangles, close together
|
||||
cell_h_od = 0.13
|
||||
n_od = 6
|
||||
y_od_ctr = 3.25
|
||||
y_os_ctr = 1.75
|
||||
_vector(ax, 0.65, y_od_ctr, n_od, cell_h=cell_h_od, cell_w=0.28, color=C_BRIDGE)
|
||||
_text(ax, 0.65, 4.10, "z_OD", fontsize=7.2, color="#333")
|
||||
_vector(ax, 0.65, y_os_ctr, n_od, cell_h=cell_h_od, cell_w=0.28, color=C_BRIDGE)
|
||||
_text(ax, 0.65, 0.90, "z_OS", fontsize=7.2, color="#333")
|
||||
|
||||
# Concat bracket [ on the LEFT of the two stacks, grouping them
|
||||
y_od_top = y_od_ctr + (n_od * cell_h_od + (n_od - 1) * 0.02) / 2
|
||||
y_os_bot = y_os_ctr - (n_od * cell_h_od + (n_od - 1) * 0.02) / 2
|
||||
x_bracket = 0.42
|
||||
ax.plot([x_bracket + 0.10, x_bracket, x_bracket, x_bracket + 0.10],
|
||||
[y_od_top, y_od_top, y_os_bot, y_os_bot],
|
||||
color="#555", lw=1.3, solid_capstyle="round", zorder=2)
|
||||
_text(ax, x_bracket - 0.06, (y_od_top + y_os_bot) / 2, "cat",
|
||||
fontsize=7.0, color="white", ha="right", va="center")
|
||||
ax.text(x_bracket - 0.02, (y_od_top + y_os_bot) / 2, "cat",
|
||||
ha="right", va="center", fontsize=7.0, color="white",
|
||||
fontfamily=FONT, fontweight="bold", zorder=6,
|
||||
bbox=dict(facecolor="#555", edgecolor="none", pad=2.5,
|
||||
boxstyle="round,pad=0.20"))
|
||||
|
||||
# FC 512 → 256 as a column of circles
|
||||
cx1, n1 = 2.10, 6
|
||||
y_top1 = 2.50 + (n1 * 0.30) / 2 - 0.15
|
||||
circle_ys_1 = [y_top1 - j * 0.30 for j in range(n1)]
|
||||
for y in circle_ys_1:
|
||||
ax.add_patch(plt.Circle((cx1, y), 0.10, facecolor=C_HEAD,
|
||||
edgecolor="white", lw=0.6, alpha=0.9, zorder=3))
|
||||
_text(ax, cx1, 0.85, "FC\n512→256", fontsize=6.8, color="#333")
|
||||
|
||||
# Dense connections from every z_OD and z_OS cell to every FC circle
|
||||
cell_step = cell_h_od + 0.02
|
||||
od_cell_ys = [y_od_ctr + (n_od * cell_h_od + (n_od - 1) * 0.02) / 2
|
||||
- cell_h_od / 2 - j * cell_step for j in range(n_od)]
|
||||
os_cell_ys = [y_os_ctr + (n_od * cell_h_od + (n_od - 1) * 0.02) / 2
|
||||
- cell_h_od / 2 - j * cell_step for j in range(n_od)]
|
||||
for y_src in od_cell_ys + os_cell_ys:
|
||||
for y_dst in circle_ys_1:
|
||||
ax.plot([0.65 + 0.14, cx1 - 0.10], [y_src, y_dst],
|
||||
color="#888", lw=0.22, alpha=0.30, zorder=1)
|
||||
|
||||
# Arrow with ReLU + Drop label on the connection to the head
|
||||
_arrow(ax, cx1 + 0.15, 2.50, 3.05, 2.50, lw=0.9)
|
||||
_text(ax, 2.65, 2.80, "ReLU + Drop", fontsize=6.5, color="#555")
|
||||
|
||||
# FC 256 → 2 as a shorter column of circles (2 units)
|
||||
cx2, n2 = 3.30, 2
|
||||
circle_ys_2 = [2.90 - j * 0.80 for j in range(n2)]
|
||||
for y in circle_ys_2:
|
||||
ax.add_patch(plt.Circle((cx2, y), 0.10, facecolor=C_HEAD,
|
||||
edgecolor="white", lw=0.6, alpha=0.9, zorder=3))
|
||||
_text(ax, cx2, 0.85, "FC\n256→2", fontsize=6.8, color="#333")
|
||||
|
||||
# Dense connections between the two FC circle columns
|
||||
for y0 in circle_ys_1:
|
||||
for y1 in circle_ys_2:
|
||||
ax.plot([cx1 + 0.10, cx2 - 0.10], [y0, y1],
|
||||
color="#888", lw=0.25, alpha=0.35, zorder=2)
|
||||
|
||||
# Two class output boxes
|
||||
for i, cls in enumerate(("Glauc.", "Normal")):
|
||||
y = 2.90 - i * 0.80
|
||||
_arrow(ax, cx2 + 0.10, y, 4.05, y, lw=0.7)
|
||||
_box(ax, 4.40, y, 0.65, 0.40, C_OUT, cls, fontsize=6.8, radius=0.08)
|
||||
|
||||
_text(ax, 2.50, 0.30,
|
||||
"Concat OD + OS → FC 512→256\n→ ReLU + Drop → FC 256→2 → softmax",
|
||||
fontsize=7.5, color="#555")
|
||||
|
||||
|
||||
# ── Main figure ──────────────────────────────────────────────────────────────
|
||||
|
||||
def main() -> None:
|
||||
def _draw_main(ax) -> None:
|
||||
W, H = 13.0, 7.8
|
||||
fig, ax = plt.subplots(figsize=(W, H))
|
||||
ax.set_xlim(0, W); ax.set_ylim(0, H); ax.axis("off")
|
||||
ax.set_facecolor(C_BG); fig.patch.set_facecolor(C_BG)
|
||||
ax.set_xlim(0, W); ax.set_ylim(0, H)
|
||||
ax.set_xticks([]); ax.set_yticks([])
|
||||
for spine in ax.spines.values():
|
||||
spine.set_visible(False)
|
||||
ax.set_facecolor(C_BG)
|
||||
|
||||
ax.set_title("Bilateral Multimodal Fusion Architecture",
|
||||
fontsize=13, fontweight="bold", fontfamily=FONT, pad=10, color="#222")
|
||||
@@ -171,14 +456,41 @@ def main() -> None:
|
||||
_arrow(ax, br_od_x + 0.05, cy_od, head_x - head_w / 2, head_y, lw=1.4, style="-|>")
|
||||
_arrow(ax, br_os_x + 0.05, cy_os, head_x - head_w / 2, head_y, lw=1.4, style="-|>")
|
||||
_box(ax, head_x, head_y, head_w, head_h, C_HEAD,
|
||||
"Patient Head\ncat(z_OD, z_OS)\n→ FC(256) → FC(2)",
|
||||
"Patient-Level Fusion\n(Concatenation)",
|
||||
fontsize=8.5, radius=0.10)
|
||||
_tag(ax, head_x + head_w / 2 - 0.13, head_y + head_h / 2 - 0.13, "D", color=C_HEAD)
|
||||
|
||||
# Output nodes
|
||||
out_x = head_x + head_w / 2 + 0.55
|
||||
_arrow(ax, head_x + head_w / 2, head_y, out_x, head_y, lw=1.5, style="-|>")
|
||||
_draw_output(ax, out_x, head_y)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
fig = plt.figure(figsize=(18.0, 8.0))
|
||||
fig.patch.set_facecolor(C_BG)
|
||||
|
||||
outer = fig.add_gridspec(
|
||||
1, 2,
|
||||
width_ratios=[12.0, 6.0],
|
||||
wspace=0.02,
|
||||
left=0.01, right=0.995, top=0.985, bottom=0.02,
|
||||
)
|
||||
|
||||
ax_main = fig.add_subplot(outer[0, 0])
|
||||
_draw_main(ax_main)
|
||||
|
||||
# 2x2 grid of component details to the right of the main diagram.
|
||||
details = outer[0, 1].subgridspec(2, 2, wspace=0.0, hspace=0.0)
|
||||
ax_cnn = fig.add_subplot(details[0, 0])
|
||||
ax_mlp = fig.add_subplot(details[0, 1])
|
||||
ax_eye = fig.add_subplot(details[1, 0])
|
||||
ax_pat = fig.add_subplot(details[1, 1])
|
||||
_draw_cnn_detail(ax_cnn)
|
||||
_draw_mlp_detail(ax_mlp)
|
||||
_draw_eye_fusion_detail(ax_eye)
|
||||
_draw_patient_fusion_detail(ax_pat)
|
||||
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(OUT, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
|
||||
@@ -0,0 +1,399 @@
|
||||
"""F3 (main text) - Clinical, Image, Hadamard L1 fusion under the pruned clinical panel.
|
||||
|
||||
Two-row, three-column figure focused on the headline architecture (Hadamard L1
|
||||
fusion) and its two single-modality references. All three columns share the
|
||||
pruned clinical panel: astigmatism, dioptre_1, dioptre_2, and Phakic/
|
||||
Pseudophakic dropped (see S8e for the fused-head permutation importance that
|
||||
motivated the prune; the paired-rep effect of dropping them is nil at fusion
|
||||
and +5 AUC pts at clinical-only). The image column has no clinical inputs so
|
||||
is unaffected by the prune.
|
||||
|
||||
Columns (left -> right):
|
||||
Clinical only (cd_solo_single_dropzero, cd_fuse)
|
||||
Image only (img_solo_single_refugelike, img_fuse)
|
||||
Hadamard L1 fusion (ensemble_single_refugelike_dropzero, nt)
|
||||
|
||||
Rows:
|
||||
top: confidence strip - predicted P(Glaucoma) coloured by VF-MD tier
|
||||
(severe / moderate / early) with normals in grey. Legend AUC is
|
||||
the per-fold-rep mean +/- SD across the 50 fold-reps.
|
||||
bottom: ROC per severity tier, each tier vs all normals; pooled ROC curve
|
||||
plotted on top of a shaded band showing the per-fold-rep TPR
|
||||
mean +/- SD at each FPR grid point; legend AUC is per-fold-rep
|
||||
mean +/- SD.
|
||||
|
||||
Glaucoma - VF_MD not recorded is dropped from both rows (n = 0 patients at
|
||||
the patient-worst-eye level).
|
||||
|
||||
Re-run:
|
||||
python -m v4.figures.F3_hadamard_focus
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import h5py
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.patches as mpatches
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.metrics import roc_auc_score, roc_curve
|
||||
|
||||
from v4.figures.util.loaders import REPO_ROOT, RESULTS_ROOT
|
||||
|
||||
OUT = Path(__file__).parent / "output" / "F3_hadamard_focus.png"
|
||||
|
||||
# ── Palette (matches other manuscript figures) ────────────────────────────────
|
||||
C_NORMAL = "#78909C"
|
||||
C_EARLY = "#29B6F6"
|
||||
C_MODERATE = "#FFB300"
|
||||
C_SEVERE = "#E53935"
|
||||
|
||||
SEV_LABELS = {
|
||||
"normal": "Normal",
|
||||
"early": "Glaucoma - early (VF_MD > -6)",
|
||||
"moderate": "Glaucoma - moderate (-12 to -6)",
|
||||
"severe": "Glaucoma - severe (VF_MD <= -12)",
|
||||
}
|
||||
SEV_COLORS = {
|
||||
"normal": C_NORMAL,
|
||||
"early": C_EARLY,
|
||||
"moderate": C_MODERATE,
|
||||
"severe": C_SEVERE,
|
||||
}
|
||||
STRIP_ORDER = ["severe", "moderate", "early", "normal"]
|
||||
ROC_ORDER = ["severe", "moderate", "early"]
|
||||
SEV_ALPHA = 0.55
|
||||
SEV_SIZE = 8
|
||||
|
||||
# Class-density violin fill
|
||||
C_NORMAL_VIOLIN = "#4c72b0"
|
||||
C_GLAUCOMA_VIOLIN = "#c44e52"
|
||||
|
||||
# ── Sources ───────────────────────────────────────────────────────────────────
|
||||
SOURCES = [
|
||||
("Clinical only",
|
||||
RESULTS_ROOT / "explainability" / "cd_solo_single_dropzero",
|
||||
"cd_fuse"),
|
||||
("Image only",
|
||||
RESULTS_ROOT / "refuge_v2m_baseline" / "img_solo_single_refugelike",
|
||||
"img_fuse"),
|
||||
("Hadamard L1 fusion",
|
||||
RESULTS_ROOT / "explainability" / "ensemble_single_refugelike_dropzero",
|
||||
"nt"),
|
||||
]
|
||||
|
||||
CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
|
||||
|
||||
# Common FPR grid for per-fold-rep ROC interpolation.
|
||||
FPR_GRID = np.linspace(0.0, 1.0, 201)
|
||||
|
||||
|
||||
def _per_foldrep_auc_and_band(
|
||||
neg_df: pd.DataFrame, pos_df: pd.DataFrame,
|
||||
) -> tuple[float, float, np.ndarray, np.ndarray]:
|
||||
"""Return (auc_mean, auc_sd, tpr_mean, tpr_sd) computed across fold-reps.
|
||||
|
||||
Each (rep, fold) is one observation: build an ROC on its own negatives +
|
||||
positives, interpolate TPR to FPR_GRID, record its AUC. Report the mean
|
||||
and standard deviation across the 50 fold-reps. Fold-reps whose subset
|
||||
has only one class present (rare, e.g. a tier missing from a fold) are
|
||||
dropped from that tier's calculation.
|
||||
"""
|
||||
nan_curve = np.full_like(FPR_GRID, np.nan, dtype=float)
|
||||
if len(pos_df) == 0 or len(neg_df) == 0:
|
||||
return float("nan"), float("nan"), nan_curve, nan_curve
|
||||
|
||||
keys = sorted(set(zip(neg_df["rep"], neg_df["fold"])) |
|
||||
set(zip(pos_df["rep"], pos_df["fold"])))
|
||||
aucs: list[float] = []
|
||||
tprs: list[np.ndarray] = []
|
||||
for rep, fold in keys:
|
||||
n = neg_df[(neg_df["rep"] == rep) & (neg_df["fold"] == fold)]
|
||||
p = pos_df[(pos_df["rep"] == rep) & (pos_df["fold"] == fold)]
|
||||
if len(n) == 0 or len(p) == 0:
|
||||
continue
|
||||
y = np.concatenate([np.zeros(len(n), dtype=int),
|
||||
np.ones(len(p), dtype=int)])
|
||||
s = np.concatenate([n["prob_glaucoma"].values,
|
||||
p["prob_glaucoma"].values])
|
||||
if len(np.unique(y)) < 2:
|
||||
continue
|
||||
fpr, tpr, _ = roc_curve(y, s)
|
||||
# roc_curve returns duplicate fpr=0 rows: (0, 0) then (0, y_first_step).
|
||||
# np.interp resolves the tie to the last value, which lifts the curve
|
||||
# off the origin. Force the origin to (0, 0) so the mean ROC actually
|
||||
# starts where it should.
|
||||
tpr_i = np.interp(FPR_GRID, fpr, tpr)
|
||||
tpr_i[FPR_GRID <= 0.0] = 0.0
|
||||
tprs.append(tpr_i)
|
||||
aucs.append(roc_auc_score(y, s))
|
||||
|
||||
if not aucs:
|
||||
return float("nan"), float("nan"), nan_curve, nan_curve
|
||||
aucs_a = np.array(aucs)
|
||||
tprs_a = np.stack(tprs, axis=0)
|
||||
return (float(aucs_a.mean()),
|
||||
float(aucs_a.std(ddof=1)) if len(aucs_a) > 1 else 0.0,
|
||||
tprs_a.mean(axis=0),
|
||||
tprs_a.std(axis=0, ddof=1) if len(tprs_a) > 1
|
||||
else np.zeros_like(FPR_GRID))
|
||||
|
||||
|
||||
# ── VFI loader (patient-level worst-eye severity) ─────────────────────────────
|
||||
|
||||
def load_vfi() -> pd.DataFrame:
|
||||
od = pd.read_excel(CLINICAL_DIR / "patient_data_od.xlsx", header=1)
|
||||
os_= pd.read_excel(CLINICAL_DIR / "patient_data_os.xlsx", header=1)
|
||||
|
||||
def _clean(df):
|
||||
df = df.copy()
|
||||
if "Patient ID" not in df.columns and "ID" in df.columns:
|
||||
df.rename(columns={"ID": "Patient ID"}, inplace=True)
|
||||
df["Patient ID"] = df["Patient ID"].astype(str).str.extract(r"(\d+)")[0].astype(int)
|
||||
df["Diagnosis"] = pd.to_numeric(df["Diagnosis"], errors="coerce")
|
||||
df["VF_MD"] = pd.to_numeric(df["VF_MD"], errors="coerce")
|
||||
df = df[df["Diagnosis"].isin([0, 1])].copy()
|
||||
return df[["Patient ID", "Diagnosis", "VF_MD"]]
|
||||
|
||||
both = pd.concat([_clean(od), _clean(os_)], ignore_index=True)
|
||||
diag = both.groupby("Patient ID")["Diagnosis"].agg(lambda x: x.mode().iloc[0]).reset_index()
|
||||
vf = both.groupby("Patient ID")["VF_MD"].min().reset_index()
|
||||
out = diag.merge(vf, on="Patient ID").rename(
|
||||
columns={"Patient ID": "patient_id", "Diagnosis": "diagnosis", "VF_MD": "vf_md"}
|
||||
)
|
||||
|
||||
def _sev(row):
|
||||
if int(row["diagnosis"]) == 0: return "normal"
|
||||
v = row["vf_md"]
|
||||
if pd.isna(v): return "unknown"
|
||||
if v > -6: return "early"
|
||||
if v > -12: return "moderate"
|
||||
return "severe"
|
||||
out["severity"] = out.apply(_sev, axis=1)
|
||||
return out
|
||||
|
||||
|
||||
# ── Prediction pooler ─────────────────────────────────────────────────────────
|
||||
|
||||
def collect_predictions(run_dir: Path, eval_stage: str) -> pd.DataFrame:
|
||||
if not run_dir.exists():
|
||||
return pd.DataFrame()
|
||||
rows: list[dict] = []
|
||||
for rep in sorted(run_dir.glob("rep*")):
|
||||
fp = next(iter(rep.rglob("predictions.h5")), None)
|
||||
if fp is None: continue
|
||||
with h5py.File(fp, "r") as f:
|
||||
if eval_stage not in f: continue
|
||||
grp = f[eval_stage]
|
||||
logits = grp["logits"][:]
|
||||
y_true = grp["y_true"][:].astype(int)
|
||||
split = grp["split"][:]
|
||||
eid0 = grp["entity_id_0"][:]
|
||||
n_folds, n_epochs, _, n_heads, n_outputs = logits.shape
|
||||
if n_outputs != 2: continue
|
||||
ep, head = n_epochs - 1, n_heads - 1
|
||||
for fold in range(n_folds):
|
||||
labels = np.array([s.decode() if isinstance(s, bytes) else str(s) for s in split[fold]])
|
||||
test_mask = labels == "test"
|
||||
if not test_mask.any(): continue
|
||||
# The 'logits' field in predictions.h5 is misnamed: the values
|
||||
# already sum to 1 across the output axis (i.e. they are softmax
|
||||
# probabilities). Use column 1 directly as P(Glaucoma); applying
|
||||
# another softmax here would compress everything into
|
||||
# [sigmoid(-1), sigmoid(1)] = [0.269, 0.731] (rank preserved so
|
||||
# AUC is unchanged, but probabilities and thresholds become
|
||||
# meaningless).
|
||||
p = logits[fold, ep, test_mask, head, :]
|
||||
for k, idx in enumerate(np.where(test_mask)[0]):
|
||||
rows.append({
|
||||
"rep": rep.name,
|
||||
"fold": fold,
|
||||
"patient_id": int(eid0[idx]),
|
||||
"y_true": int(y_true[idx]),
|
||||
"prob_glaucoma": float(p[k, 1]),
|
||||
})
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
# ── Confidence strip ──────────────────────────────────────────────────────────
|
||||
|
||||
def _merge_and_filter(df: pd.DataFrame, vfi: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Attach patient-level severity and apply the consistent-label filter.
|
||||
|
||||
Two exclusions:
|
||||
1. Patients whose patient-level severity is 'unknown' (glaucoma without
|
||||
a recorded VF_MD; 0 patients in PAPILA at the patient level).
|
||||
2. Mixed-label predictions where the model's eye-level label is
|
||||
glaucoma (y_true == 1) but the patient's severity resolves to
|
||||
'normal' via the load_vfi mode-of-both-eyes rule. ~70 predictions
|
||||
in this dataset; excluding them makes the strip-corner AUC equal to
|
||||
the sample-size-weighted mean of the per-tier ROC AUCs.
|
||||
"""
|
||||
df = df.merge(vfi[["patient_id", "severity"]], on="patient_id", how="left")
|
||||
df["severity"] = df["severity"].fillna("unknown")
|
||||
df = df[df["severity"] != "unknown"]
|
||||
df = df[~((df["y_true"] == 1) & (df["severity"] == "normal"))]
|
||||
return df.copy()
|
||||
|
||||
|
||||
def _draw_strip(ax, df: pd.DataFrame, vfi: pd.DataFrame, label: str):
|
||||
df = _merge_and_filter(df, vfi)
|
||||
|
||||
rng = np.random.default_rng(42)
|
||||
x_pos = {0: 0.0, 1: 1.0}
|
||||
jitter_scale = 0.18
|
||||
|
||||
data_by_class = [df.loc[df["y_true"] == cls, "prob_glaucoma"].values for cls in [0, 1]]
|
||||
if all(len(d) > 0 for d in data_by_class):
|
||||
vp = ax.violinplot(data_by_class, positions=[0, 1], widths=0.7,
|
||||
showmedians=False, showextrema=False)
|
||||
for body, color in zip(vp["bodies"], [C_NORMAL_VIOLIN, C_GLAUCOMA_VIOLIN]):
|
||||
body.set_facecolor(color); body.set_alpha(0.30)
|
||||
body.set_edgecolor("none"); body.set_zorder(2)
|
||||
|
||||
for sev in STRIP_ORDER:
|
||||
mask = df["severity"] == sev
|
||||
if not mask.any(): continue
|
||||
sub = df[mask]
|
||||
jitter = rng.uniform(-jitter_scale, jitter_scale, len(sub))
|
||||
x = np.array([x_pos[int(v)] for v in sub["y_true"]]) + jitter
|
||||
ax.scatter(x, sub["prob_glaucoma"].values,
|
||||
c=SEV_COLORS[sev], s=SEV_SIZE,
|
||||
alpha=SEV_ALPHA, linewidths=0, zorder=3)
|
||||
|
||||
xtick_labels = []
|
||||
for cls, xc in x_pos.items():
|
||||
vals = df.loc[df["y_true"] == cls, "prob_glaucoma"]
|
||||
if not len(vals):
|
||||
xtick_labels.append("Normal" if cls == 0 else "Glaucoma"); continue
|
||||
med = float(np.median(vals))
|
||||
ax.plot([xc - jitter_scale - 0.04, xc + jitter_scale + 0.04],
|
||||
[med, med], color="#222", lw=2.0, zorder=5)
|
||||
if cls == 0:
|
||||
rate = (vals <= 0.5).mean() * 100
|
||||
xtick_labels.append(f"Normal\nTN {rate:.0f}%")
|
||||
else:
|
||||
rate = (vals > 0.5).mean() * 100
|
||||
xtick_labels.append(f"Glaucoma\nTP {rate:.0f}%")
|
||||
|
||||
ax.axhline(0.5, color="#888", lw=1.2, ls="--", alpha=0.7, zorder=2)
|
||||
ax.set_xticks([0, 1]); ax.set_xticklabels(xtick_labels, fontsize=10)
|
||||
ax.set_ylim(-0.04, 1.04); ax.set_xlim(-0.55, 1.55)
|
||||
ax.set_title(label, fontsize=11, fontweight="bold")
|
||||
ax.grid(axis="y", alpha=0.3, zorder=1)
|
||||
|
||||
# Per-fold-rep AUC mean +/- SD across the 50 fold-reps. Uses the same
|
||||
# computation as the ROC panel legend for consistency.
|
||||
neg_df = df[df["y_true"] == 0]
|
||||
pos_df = df[df["y_true"] == 1]
|
||||
if len(pos_df) > 0 and len(neg_df) > 0:
|
||||
auc_mean, auc_sd, *_ = _per_foldrep_auc_and_band(neg_df, pos_df)
|
||||
ax.text(0.98, 0.02,
|
||||
f"AUC = {auc_mean:.3f} +/- {auc_sd:.3f}",
|
||||
transform=ax.transAxes, ha="right", va="bottom",
|
||||
fontsize=9, color="#333",
|
||||
bbox=dict(facecolor="white", alpha=0.75, edgecolor="none", pad=2))
|
||||
|
||||
|
||||
# ── ROC by severity ───────────────────────────────────────────────────────────
|
||||
|
||||
def _draw_roc(ax, df: pd.DataFrame, vfi: pd.DataFrame, label: str):
|
||||
df = _merge_and_filter(df, vfi)
|
||||
|
||||
# Diagonal reference behind everything
|
||||
ax.plot([0, 1], [0, 1], color="#aaa", linewidth=0.7, linestyle=":", zorder=1)
|
||||
|
||||
for tier in ROC_ORDER:
|
||||
neg = df[df["y_true"] == 0]
|
||||
pos = df[(df["y_true"] == 1) & (df["severity"] == tier)]
|
||||
if len(pos) == 0 or len(neg) == 0:
|
||||
continue
|
||||
|
||||
# Per-fold-rep mean ROC (plotted on FPR_GRID) with a +/- SD band and
|
||||
# legend AUC as per-fold-rep mean +/- SD.
|
||||
auc_mean, auc_sd, tpr_mean, tpr_sd = _per_foldrep_auc_and_band(neg, pos)
|
||||
tpr_lo = np.clip(tpr_mean - tpr_sd, 0.0, 1.0)
|
||||
tpr_hi = np.clip(tpr_mean + tpr_sd, 0.0, 1.0)
|
||||
|
||||
color = SEV_COLORS[tier]
|
||||
ax.fill_between(FPR_GRID, tpr_lo, tpr_hi,
|
||||
color=color, alpha=0.16, linewidth=0, zorder=2)
|
||||
label_txt = f"{tier.capitalize()} AUC = {auc_mean:.3f} +/- {auc_sd:.3f}"
|
||||
ax.plot(FPR_GRID, tpr_mean, color=color, linewidth=2.0, zorder=3, label=label_txt)
|
||||
|
||||
ax.set_xlim(0, 1); ax.set_ylim(0, 1.02)
|
||||
ax.set_xlabel("False positive rate", fontsize=10)
|
||||
ax.grid(alpha=0.25, linestyle="--", zorder=1)
|
||||
ax.legend(loc="lower right", fontsize=8.5, framealpha=0.94)
|
||||
ax.set_title(label, fontsize=11, fontweight="bold")
|
||||
|
||||
|
||||
# ── Render ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def render() -> None:
|
||||
vfi = load_vfi()
|
||||
frames = [(lbl, collect_predictions(p, s)) for lbl, p, s in SOURCES]
|
||||
for lbl, df in frames:
|
||||
if len(df):
|
||||
print(f" {lbl:<20s} n_rows={len(df):>5d} (pid={df['patient_id'].nunique()}, reps={df['rep'].nunique()})")
|
||||
else:
|
||||
print(f" {lbl:<20s} no data")
|
||||
|
||||
n_cols = len(SOURCES)
|
||||
fig = plt.figure(figsize=(4.6 * n_cols, 10.4))
|
||||
fig.patch.set_facecolor("#e8e8e8")
|
||||
|
||||
gs = fig.add_gridspec(2, n_cols, hspace=0.30, wspace=0.16,
|
||||
left=0.06, right=0.985, top=0.94, bottom=0.10)
|
||||
|
||||
strip_axes, roc_axes = [], []
|
||||
strip_sharey, roc_sharey = None, None
|
||||
for i, (lbl, df) in enumerate(frames):
|
||||
ax_s = fig.add_subplot(gs[0, i], sharey=strip_sharey)
|
||||
ax_r = fig.add_subplot(gs[1, i], sharey=roc_sharey)
|
||||
strip_sharey = strip_sharey or ax_s
|
||||
roc_sharey = roc_sharey or ax_r
|
||||
strip_axes.append(ax_s); roc_axes.append(ax_r)
|
||||
ax_s.set_facecolor("#e8e8e8"); ax_r.set_facecolor("#e8e8e8")
|
||||
if len(df):
|
||||
_draw_strip(ax_s, df, vfi, lbl)
|
||||
_draw_roc(ax_r, df, vfi, lbl)
|
||||
else:
|
||||
for ax in (ax_s, ax_r):
|
||||
ax.text(0.5, 0.5, "(pending)", ha="center", va="center",
|
||||
fontsize=12, color="#888", transform=ax.transAxes)
|
||||
ax.set_xticks([]); ax.set_yticks([])
|
||||
ax.set_title(lbl, fontsize=11, fontweight="bold")
|
||||
|
||||
strip_axes[0].set_ylabel("Predicted P(Glaucoma)", fontsize=11)
|
||||
roc_axes[0].set_ylabel("True positive rate", fontsize=11)
|
||||
# Force numeric ticks on the leftmost panel of each row (matplotlib
|
||||
# sometimes drops them under sharey after set_ylim); hide on the rest.
|
||||
strip_axes[0].set_yticks(np.linspace(0, 1, 6))
|
||||
roc_axes[0].set_yticks(np.linspace(0, 1, 6))
|
||||
strip_axes[0].tick_params(axis="y", labelleft=True)
|
||||
roc_axes[0].tick_params(axis="y", labelleft=True)
|
||||
for ax in strip_axes[1:] + roc_axes[1:]:
|
||||
ax.tick_params(axis="y", labelleft=False)
|
||||
|
||||
# Shared severity legend (strip row) at figure bottom
|
||||
legend_patches = [mpatches.Patch(color=SEV_COLORS[s], label=SEV_LABELS[s])
|
||||
for s in ["normal", "early", "moderate", "severe"]]
|
||||
fig.legend(handles=legend_patches, fontsize=9,
|
||||
loc="lower center", ncol=len(legend_patches),
|
||||
framealpha=0.75, bbox_to_anchor=(0.5, 0.0))
|
||||
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(OUT, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"saved {OUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
render()
|
||||
+38
-318
@@ -1,344 +1,64 @@
|
||||
"""F4 — Bilateral lift via confidence strips.
|
||||
"""F4 (main text) - Bilateral clinical, image, and Hadamard L2 fusion.
|
||||
|
||||
Six-panel grid in the same v3-style as F3. Rows are aggregation mode,
|
||||
columns are tower configuration:
|
||||
Patient-level analogue of F3: same two-row / three-column layout, same pruned
|
||||
clinical panel, but every column is the bilateral (both-eyes) model reading
|
||||
from the patient-level 'hb' stage. All three columns share the pruned clinical
|
||||
panel: astigmatism, dioptre_1, dioptre_2, and Phakic/Pseudophakic dropped
|
||||
(see S8e). The image column has no clinical inputs so is unaffected by the
|
||||
prune.
|
||||
|
||||
Clinical Image Fusion (img+cd)
|
||||
Single | cd_solo_single | img_solo_single | ensemble_single (Hadamard L1)
|
||||
Bilateral| cd_solo_bilat | img_solo_bilat | baseline_ensemble (L2 concat default)
|
||||
Columns (left -> right):
|
||||
Clinical only (bilateral) (cd_solo_bilateral_dropzero, hb)
|
||||
Image only (bilateral) (img_solo_bilateral_refugelike, hb)
|
||||
Hadamard L2 fusion (bilateral) (ensemble_refugelike_ckpt_dropzero, hb)
|
||||
|
||||
All refugelike. Single-eye panels eval at the appropriate eye-level fusion
|
||||
stage; bilateral panels eval at hb. Points are coloured by VF-MD severity.
|
||||
Each panel shows per-patient or per-eye P(Glaucoma) with TN / TP rates and
|
||||
AUC printed in.
|
||||
Rows:
|
||||
top: confidence strip - predicted P(Glaucoma) coloured by VF-MD tier
|
||||
(severe / moderate / early) with normals in grey. Patients are the
|
||||
unit of prediction here (~1 prediction per patient per fold-rep).
|
||||
bottom: ROC per severity tier, each tier vs all normals; pooled ROC curve
|
||||
with a 95% CI band from patient-level bootstrap; pooled AUC with
|
||||
95% CI annotated.
|
||||
|
||||
Re-run anytime:
|
||||
Glaucoma - VF_MD not recorded is dropped from both rows (n = 0 patients at
|
||||
the patient-worst-eye level).
|
||||
|
||||
Re-run:
|
||||
python -m v4.figures.F4_bilateral
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import h5py
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.patches as mpatches
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.metrics import roc_auc_score
|
||||
|
||||
from v4.figures.util.loaders import REPO_ROOT, RESULTS_ROOT
|
||||
from v4.figures.util.loaders import RESULTS_ROOT
|
||||
from v4.figures import F3_hadamard_focus as F3
|
||||
|
||||
OUT = Path(__file__).parent / "output" / "F4_bilateral.png"
|
||||
|
||||
# Palette (matches F3) ───────────────────────────────────────────────────────
|
||||
C_NORMAL = "#78909C"
|
||||
C_EARLY = "#29B6F6"
|
||||
C_MODERATE = "#FFB300"
|
||||
C_SEVERE = "#E53935"
|
||||
C_UNKNOWN = "#BDBDBD"
|
||||
SEV_COLORS = {
|
||||
"normal": C_NORMAL,
|
||||
"early": C_EARLY,
|
||||
"moderate": C_MODERATE,
|
||||
"severe": C_SEVERE,
|
||||
"unknown": C_UNKNOWN,
|
||||
}
|
||||
SEV_LABELS = {
|
||||
"normal": "Normal",
|
||||
"early": "Glaucoma — early (VF_MD > −6)",
|
||||
"moderate": "Glaucoma — moderate (−12 to −6)",
|
||||
"severe": "Glaucoma — severe (VF_MD < −12)",
|
||||
"unknown": "Glaucoma — VF_MD not recorded",
|
||||
}
|
||||
SEV_ORDER = ["severe", "moderate", "unknown", "early", "normal"]
|
||||
SEV_ALPHA = {
|
||||
"normal": 0.55,
|
||||
"unknown": 0.55,
|
||||
"early": 0.55,
|
||||
"moderate": 0.55,
|
||||
"severe": 0.55,
|
||||
}
|
||||
SEV_SIZE = {"normal": 8, "unknown": 8, "early": 8, "moderate": 8, "severe": 8}
|
||||
|
||||
# Panel grid: [row][col] = (label, run_dir, eval_stage)
|
||||
GRID = [
|
||||
[
|
||||
("Single · Clinical", RESULTS_ROOT / "phase2_v4" / "cd_solo_single", "cd_fuse"),
|
||||
(
|
||||
"Single · Image",
|
||||
RESULTS_ROOT / "refuge_v2m_baseline" / "img_solo_single_refugelike",
|
||||
"img_fuse",
|
||||
),
|
||||
(
|
||||
"Single · Fusion",
|
||||
RESULTS_ROOT / "refuge_v2m_baseline" / "ensemble_single_refugelike",
|
||||
"nt",
|
||||
),
|
||||
],
|
||||
[
|
||||
(
|
||||
"Bilateral · Clinical",
|
||||
RESULTS_ROOT / "phase4_v4" / "cd_solo_bilateral",
|
||||
"hb",
|
||||
),
|
||||
(
|
||||
"Bilateral · Image",
|
||||
RESULTS_ROOT / "refuge_v2m_baseline" / "img_solo_bilateral_refugelike",
|
||||
"hb",
|
||||
),
|
||||
("Bilateral · Fusion", RESULTS_ROOT / "tri_v1" / "baseline_ensemble", "hb"),
|
||||
],
|
||||
SOURCES = [
|
||||
("Clinical only (bilateral)",
|
||||
RESULTS_ROOT / "explainability" / "cd_solo_bilateral_dropzero",
|
||||
"hb"),
|
||||
("Image only (bilateral)",
|
||||
RESULTS_ROOT / "refuge_v2m_baseline" / "img_solo_bilateral_refugelike",
|
||||
"hb"),
|
||||
("Hadamard L2 fusion (bilateral)",
|
||||
RESULTS_ROOT / "explainability" / "ensemble_refugelike_ckpt_dropzero",
|
||||
"hb"),
|
||||
]
|
||||
|
||||
CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
|
||||
|
||||
|
||||
# ── Same helpers as F3 ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def load_vfi() -> pd.DataFrame:
|
||||
od = pd.read_excel(CLINICAL_DIR / "patient_data_od.xlsx", header=1)
|
||||
os_ = pd.read_excel(CLINICAL_DIR / "patient_data_os.xlsx", header=1)
|
||||
|
||||
def _clean(df):
|
||||
df = df.copy()
|
||||
if "Patient ID" not in df.columns and "ID" in df.columns:
|
||||
df.rename(columns={"ID": "Patient ID"}, inplace=True)
|
||||
df["Patient ID"] = (
|
||||
df["Patient ID"].astype(str).str.extract(r"(\d+)")[0].astype(int)
|
||||
)
|
||||
df["Diagnosis"] = pd.to_numeric(df["Diagnosis"], errors="coerce")
|
||||
df["VF_MD"] = pd.to_numeric(df["VF_MD"], errors="coerce")
|
||||
df = df[df["Diagnosis"].isin([0, 1])].copy()
|
||||
return df[["Patient ID", "Diagnosis", "VF_MD"]]
|
||||
|
||||
both = pd.concat([_clean(od), _clean(os_)], ignore_index=True)
|
||||
diag = (
|
||||
both.groupby("Patient ID")["Diagnosis"]
|
||||
.agg(lambda x: x.mode().iloc[0])
|
||||
.reset_index()
|
||||
)
|
||||
vf = both.groupby("Patient ID")["VF_MD"].min().reset_index()
|
||||
out = diag.merge(vf, on="Patient ID").rename(
|
||||
columns={"Patient ID": "patient_id", "Diagnosis": "diagnosis", "VF_MD": "vf_md"}
|
||||
)
|
||||
|
||||
def _sev(row):
|
||||
if int(row["diagnosis"]) == 0:
|
||||
return "normal"
|
||||
v = row["vf_md"]
|
||||
if pd.isna(v):
|
||||
return "unknown"
|
||||
if v > -6:
|
||||
return "early"
|
||||
if v > -12:
|
||||
return "moderate"
|
||||
return "severe"
|
||||
|
||||
out["severity"] = out.apply(_sev, axis=1)
|
||||
return out
|
||||
|
||||
|
||||
def collect_predictions(run_dir: Path, eval_stage: str) -> pd.DataFrame:
|
||||
if not run_dir.exists():
|
||||
return pd.DataFrame()
|
||||
rows: list[dict] = []
|
||||
for rep in sorted(run_dir.glob("rep*")):
|
||||
fp = next(iter(rep.rglob("predictions.h5")), None)
|
||||
if fp is None:
|
||||
continue
|
||||
with h5py.File(fp, "r") as f:
|
||||
if eval_stage not in f:
|
||||
continue
|
||||
grp = f[eval_stage]
|
||||
logits = grp["logits"][:]
|
||||
y_true = grp["y_true"][:].astype(int)
|
||||
split = grp["split"][:]
|
||||
eid0 = grp["entity_id_0"][:]
|
||||
n_folds, n_epochs, n_samples, n_heads, n_outputs = logits.shape
|
||||
if n_outputs != 2:
|
||||
continue
|
||||
ep, head = n_epochs - 1, n_heads - 1
|
||||
for fold in range(n_folds):
|
||||
labels = np.array(
|
||||
[
|
||||
s.decode() if isinstance(s, bytes) else str(s)
|
||||
for s in split[fold]
|
||||
]
|
||||
)
|
||||
test_mask = labels == "test"
|
||||
if not test_mask.any():
|
||||
continue
|
||||
lg = logits[fold, ep, test_mask, head, :]
|
||||
e = np.exp(lg - lg.max(axis=1, keepdims=True))
|
||||
p = e / e.sum(axis=1, keepdims=True)
|
||||
for k, idx in enumerate(np.where(test_mask)[0]):
|
||||
rows.append(
|
||||
{
|
||||
"rep": rep.name,
|
||||
"fold": fold,
|
||||
"patient_id": int(eid0[idx]),
|
||||
"y_true": int(y_true[idx]),
|
||||
"prob_glaucoma": float(p[k, 1]),
|
||||
}
|
||||
)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
C_NORMAL_VIOLIN = "#4c72b0"
|
||||
C_GLAUCOMA_VIOLIN = "#c44e52"
|
||||
|
||||
|
||||
def _draw_panel(ax, df: pd.DataFrame, vfi: pd.DataFrame, label: str):
|
||||
df = df.merge(vfi[["patient_id", "severity"]], on="patient_id", how="left")
|
||||
df["severity"] = df["severity"].fillna("unknown")
|
||||
rng = np.random.default_rng(42)
|
||||
x_pos = {0: 0.0, 1: 1.0}
|
||||
jitter_scale = 0.18
|
||||
|
||||
# Violin density behind everything (per true class)
|
||||
data_by_class = [
|
||||
df.loc[df["y_true"] == cls, "prob_glaucoma"].values for cls in [0, 1]
|
||||
]
|
||||
if all(len(d) > 0 for d in data_by_class):
|
||||
vp = ax.violinplot(
|
||||
data_by_class,
|
||||
positions=[0, 1],
|
||||
widths=0.7,
|
||||
showmedians=False,
|
||||
showextrema=False,
|
||||
)
|
||||
for body, color in zip(vp["bodies"], [C_NORMAL_VIOLIN, C_GLAUCOMA_VIOLIN]):
|
||||
body.set_facecolor(color)
|
||||
body.set_alpha(0.30)
|
||||
body.set_edgecolor("none")
|
||||
body.set_zorder(2)
|
||||
|
||||
for sev in SEV_ORDER:
|
||||
mask = df["severity"] == sev
|
||||
if not mask.any():
|
||||
continue
|
||||
sub = df[mask]
|
||||
jitter = rng.uniform(-jitter_scale, jitter_scale, len(sub))
|
||||
x = np.array([x_pos[int(v)] for v in sub["y_true"]]) + jitter
|
||||
ax.scatter(
|
||||
x,
|
||||
sub["prob_glaucoma"].values,
|
||||
c=SEV_COLORS[sev],
|
||||
s=SEV_SIZE[sev],
|
||||
alpha=SEV_ALPHA[sev],
|
||||
linewidths=0,
|
||||
zorder=3,
|
||||
)
|
||||
xtick_labels = []
|
||||
for cls, xc in x_pos.items():
|
||||
vals = df.loc[df["y_true"] == cls, "prob_glaucoma"]
|
||||
if not len(vals):
|
||||
xtick_labels.append("Normal" if cls == 0 else "Glaucoma")
|
||||
continue
|
||||
med = float(np.median(vals))
|
||||
ax.plot(
|
||||
[xc - jitter_scale - 0.04, xc + jitter_scale + 0.04],
|
||||
[med, med],
|
||||
color="#222",
|
||||
lw=2.0,
|
||||
zorder=5,
|
||||
)
|
||||
if cls == 0:
|
||||
rate = (vals <= 0.5).mean() * 100
|
||||
xtick_labels.append(f"Normal\nTN {rate:.0f}%")
|
||||
else:
|
||||
rate = (vals > 0.5).mean() * 100
|
||||
xtick_labels.append(f"Glaucoma\nTP {rate:.0f}%")
|
||||
ax.axhline(0.5, color="#888", lw=1.2, ls="--", alpha=0.7, zorder=2)
|
||||
ax.set_xticks([0, 1])
|
||||
ax.set_xticklabels(xtick_labels, fontsize=9)
|
||||
ax.set_ylim(-0.04, 1.04)
|
||||
ax.set_xlim(-0.55, 1.55)
|
||||
ax.set_title(label, fontsize=10.5, fontweight="bold")
|
||||
ax.grid(axis="y", alpha=0.3, zorder=1)
|
||||
|
||||
fold_aucs = []
|
||||
for _, g in df.groupby(["rep", "fold"]):
|
||||
if g["y_true"].nunique() < 2:
|
||||
continue
|
||||
try:
|
||||
fold_aucs.append(roc_auc_score(g["y_true"], g["prob_glaucoma"]))
|
||||
except Exception:
|
||||
pass
|
||||
if fold_aucs:
|
||||
ax.text(
|
||||
0.66,
|
||||
0.0,
|
||||
f"AUC = {np.mean(fold_aucs):.3f} ± {np.std(fold_aucs):.3f}",
|
||||
transform=ax.transAxes,
|
||||
ha="right",
|
||||
va="bottom",
|
||||
fontsize=8.5,
|
||||
color="#333",
|
||||
bbox=dict(facecolor="white", alpha=0.7, edgecolor="none", pad=2),
|
||||
)
|
||||
|
||||
|
||||
def render() -> None:
|
||||
vfi = load_vfi()
|
||||
fig, axes = plt.subplots(2, 3, figsize=(13, 11), sharey=True)
|
||||
fig.patch.set_facecolor("#e8e8e8")
|
||||
fig.suptitle("Single → Bilateral Aggregation Lift", fontsize=13, fontweight="bold")
|
||||
|
||||
for ri, row in enumerate(GRID):
|
||||
for ci, (lbl, path, stage) in enumerate(row):
|
||||
ax = axes[ri, ci]
|
||||
ax.set_facecolor("#e8e8e8")
|
||||
df = collect_predictions(path, stage)
|
||||
if len(df):
|
||||
_draw_panel(ax, df, vfi, lbl)
|
||||
print(f" [{ri},{ci}] {lbl:<22s} n={len(df):>5d}")
|
||||
else:
|
||||
ax.text(
|
||||
0.5,
|
||||
0.5,
|
||||
"(pending)",
|
||||
ha="center",
|
||||
va="center",
|
||||
fontsize=12,
|
||||
color="#888",
|
||||
transform=ax.transAxes,
|
||||
)
|
||||
ax.set_xticks([])
|
||||
ax.set_yticks([])
|
||||
ax.set_title(lbl, fontsize=10.5, fontweight="bold")
|
||||
print(f" [{ri},{ci}] {lbl:<22s} no data yet")
|
||||
|
||||
for ri in range(2):
|
||||
axes[ri, 0].set_ylabel("Predicted P(Glaucoma)", fontsize=10.5)
|
||||
|
||||
legend_patches = [
|
||||
mpatches.Patch(color=SEV_COLORS[s], label=SEV_LABELS[s])
|
||||
for s in ["normal", "early", "moderate", "severe", "unknown"]
|
||||
]
|
||||
fig.legend(
|
||||
handles=legend_patches,
|
||||
fontsize=9,
|
||||
loc="lower center",
|
||||
ncol=len(legend_patches),
|
||||
framealpha=0.75,
|
||||
bbox_to_anchor=(0.5, -0.005),
|
||||
)
|
||||
|
||||
fig.tight_layout(rect=[0, 0.04, 1, 0.97])
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(OUT, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"saved {OUT}")
|
||||
"""Reuse every drawing primitive from F3; only source paths and out-file differ."""
|
||||
F3.SOURCES = SOURCES
|
||||
F3.OUT = OUT
|
||||
F3.render()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -447,6 +447,25 @@ def make_fusion_event_panel(split: str = "test") -> None:
|
||||
print(f"Saved: {out}")
|
||||
|
||||
|
||||
def _drop_structural_features(
|
||||
names: list[str],
|
||||
groups: list[list[int]],
|
||||
drop: set[str],
|
||||
) -> tuple[list[str], list[list[int]]]:
|
||||
"""Return (names, groups) with entries in ``drop`` removed.
|
||||
|
||||
Encoded-vector indices are left intact - the model still sees them - but
|
||||
we simply do not permute/report those dims. Used to hide index-level
|
||||
columns like eyeID that appear in the clinical feature list only because
|
||||
they double as an index level.
|
||||
"""
|
||||
kept = [(n, g) for n, g in zip(names, groups) if n not in drop]
|
||||
if not kept:
|
||||
return list(names), list(groups)
|
||||
kept_names, kept_groups = zip(*kept)
|
||||
return list(kept_names), [list(g) for g in kept_groups]
|
||||
|
||||
|
||||
def make_clinical_importance(n_permutations: int = 30, seed: int = 0) -> None:
|
||||
"""S8e clinical permutation importance via the cd-tower → cd_aux head.
|
||||
|
||||
@@ -486,6 +505,11 @@ def make_clinical_importance(n_permutations: int = 30, seed: int = 0) -> None:
|
||||
"feature_groups; skipping S8e."
|
||||
)
|
||||
return
|
||||
# eyeID is structural (used as the OD/OS index level in DataViews), not a
|
||||
# learned variable. Strip it from the reported permutation analysis.
|
||||
feature_names, feature_groups = _drop_structural_features(
|
||||
feature_names, feature_groups, drop={"eyeID"},
|
||||
)
|
||||
|
||||
label_filter = cfg.get("label_filter", None)
|
||||
df_mode = data.df.copy()
|
||||
@@ -613,7 +637,246 @@ def make_clinical_importance(n_permutations: int = 30, seed: int = 0) -> None:
|
||||
print(f"Saved: {out_csv}")
|
||||
|
||||
|
||||
def _build_v4_fold_modules(fold_idx: int, cfg: dict, data, device):
|
||||
def make_fused_clinical_importance(n_permutations: int = 30, seed: int = 0) -> None:
|
||||
"""S8e-b clinical permutation importance measured at the fused L2 head.
|
||||
|
||||
Answers a different question from the L1 (cd-only) importance: given the
|
||||
image tower is already contributing, which clinical features still change
|
||||
the patient-level fused prediction? For each permutation, the clinical
|
||||
vector is re-embedded through the cd tower, fused with cached image
|
||||
embeddings through the L1 (nt) bridge per eye, aggregated through the L2
|
||||
(hb) bridge, and scored against the patient-level label at hb_head.
|
||||
|
||||
OD and OS clinical vectors are permuted together (same source-patient
|
||||
replaces both eyes) so patient-level pairing is preserved.
|
||||
|
||||
Aggregation: all 10 reps of ``ensemble_refugelike_ckpt`` are looped; each
|
||||
rep produces a per-fold mean AUC drop, averaged within-rep to a rep-level
|
||||
drop; final bars show mean ± SD **across reps** (n=10). Baseline is the
|
||||
rep-mean of within-rep fold-mean baseline AUCs. This matches the
|
||||
rep-mean-of-fold-means aggregation used elsewhere in the manuscript.
|
||||
"""
|
||||
import json
|
||||
import torch
|
||||
import torch.nn.functional as TF_
|
||||
from tqdm import tqdm
|
||||
from v4.classes.accessory.explainability import permutation_importance
|
||||
from v4.classes.split_manager import SplitManager
|
||||
from v4.classes.v4_hypertower import load_data, _make_loader
|
||||
import v4.classes.v4_hypertower as orch
|
||||
|
||||
rep_base = V4_CKPT_RUN.parent.parent # .../ensemble_refugelike_ckpt
|
||||
rep_dirs = sorted(
|
||||
p for p in rep_base.glob("rep*/binary")
|
||||
if (p / "summary.json").exists() and (p / "checkpoints").exists()
|
||||
)
|
||||
if not rep_dirs:
|
||||
print(f"[F8] make_fused_clinical_importance: no reps found under {rep_base}")
|
||||
return
|
||||
print(f"[F8] Fused clinical importance across {len(rep_dirs)} reps")
|
||||
|
||||
cfg = json.loads((rep_dirs[0] / "summary.json").read_text())["config"]
|
||||
orch.cfg_ref = cfg
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
data = load_data(cfg)
|
||||
clinical_view = data.matrix
|
||||
feature_names = getattr(clinical_view, "feature_names", None)
|
||||
feature_groups = getattr(clinical_view, "feature_groups", None)
|
||||
if feature_names is None or feature_groups is None:
|
||||
print("[F8] make_fused_clinical_importance: profile lacks feature_names / "
|
||||
"feature_groups; skipping.")
|
||||
return
|
||||
# eyeID is structural (index level for OD/OS in DataViews), not a learned
|
||||
# variable. Skip it in the reported analysis.
|
||||
feature_names, feature_groups = _drop_structural_features(
|
||||
feature_names, feature_groups, drop={"eyeID"},
|
||||
)
|
||||
|
||||
label_filter = cfg.get("label_filter", None)
|
||||
df_mode = data.df.copy()
|
||||
if label_filter is not None:
|
||||
df_mode = df_mode[df_mode[data.label_col].isin(label_filter)].reset_index(drop=True)
|
||||
identity_cols = getattr(data, "identity_cols", [])
|
||||
identity_level = cfg.get("split_identity_level", 1)
|
||||
group_col = identity_cols[identity_level - 1] if identity_level and identity_cols else None
|
||||
splits = SplitManager(group_col=group_col).build_plans(
|
||||
df_mode,
|
||||
label_col=data.label_col,
|
||||
n_splits=cfg.get("folds", 5),
|
||||
seed=cfg.get("fold_seed", 100),
|
||||
)
|
||||
|
||||
rep_mean_drops = [] # list of (F,) arrays: within-rep fold-mean drops
|
||||
rep_mean_baselines = [] # list of floats: within-rep fold-mean baseline AUCs
|
||||
reference_names = None
|
||||
|
||||
for rep_idx, run_dir in enumerate(rep_dirs):
|
||||
rep_cfg = json.loads((run_dir / "summary.json").read_text())["config"]
|
||||
# matched-seed reps share the same fold-splitting seed / label_filter;
|
||||
# verify config alignment before reusing splits computed from cfg above.
|
||||
if rep_cfg.get("fold_seed") != cfg.get("fold_seed"):
|
||||
print(f"[F8] {run_dir.parent.name}: fold_seed mismatch, rebuilding splits.")
|
||||
splits_here = SplitManager(group_col=group_col).build_plans(
|
||||
df_mode, label_col=data.label_col,
|
||||
n_splits=rep_cfg.get("folds", 5), seed=rep_cfg.get("fold_seed", 100),
|
||||
)
|
||||
else:
|
||||
splits_here = splits
|
||||
|
||||
fold_results = []
|
||||
for fold_idx in tqdm(
|
||||
range(rep_cfg.get("folds", 5)),
|
||||
desc=f"rep {rep_idx:02d}/{len(rep_dirs)-1}",
|
||||
unit="fold",
|
||||
leave=False,
|
||||
):
|
||||
ckpt_dir = run_dir / "checkpoints" / f"fold{fold_idx}"
|
||||
if not ckpt_dir.exists():
|
||||
continue
|
||||
towers, nt, hb, img_aux, cd_aux, hb_head = _build_v4_fold_modules(
|
||||
fold_idx, rep_cfg, data, device, run_dir=run_dir,
|
||||
)
|
||||
|
||||
split_obj = splits_here[fold_idx]
|
||||
if split_obj.test is None:
|
||||
del towers, nt, hb, img_aux, cd_aux, hb_head
|
||||
continue
|
||||
shell = data.build_shells(split_obj.test, level="patient", label_filter=label_filter)
|
||||
loader = _make_loader(
|
||||
shell,
|
||||
towers,
|
||||
batch_size=rep_cfg["training"].get("batch_size", 8),
|
||||
shuffle=False,
|
||||
)
|
||||
|
||||
cd_a_list, cd_b_list = [], []
|
||||
z_img_a_list, z_img_b_list = [], []
|
||||
y_list = []
|
||||
with torch.no_grad():
|
||||
for batch in loader:
|
||||
cd_a_list.append(batch["cd"]["a"].cpu().numpy())
|
||||
cd_b_list.append(batch["cd"]["b"].cpu().numpy())
|
||||
z_img_a = towers["img"](batch["img"]["a"].to(device))
|
||||
z_img_b = towers["img"](batch["img"]["b"].to(device))
|
||||
z_img_a_list.append(z_img_a.cpu().numpy())
|
||||
z_img_b_list.append(z_img_b.cpu().numpy())
|
||||
y_list.append(batch["label"].cpu().numpy())
|
||||
if not cd_a_list:
|
||||
del towers, nt, hb, img_aux, cd_aux, hb_head
|
||||
continue
|
||||
cd_a = np.concatenate(cd_a_list).astype(np.float32)
|
||||
cd_b = np.concatenate(cd_b_list).astype(np.float32)
|
||||
z_img_a_cached = torch.from_numpy(np.concatenate(z_img_a_list)).to(device)
|
||||
z_img_b_cached = torch.from_numpy(np.concatenate(z_img_b_list)).to(device)
|
||||
y = np.concatenate(y_list).astype(int)
|
||||
if len(np.unique(y)) < 2:
|
||||
del towers, nt, hb, img_aux, cd_aux, hb_head
|
||||
continue
|
||||
|
||||
F = cd_a.shape[1]
|
||||
X_wide = np.concatenate([cd_a, cd_b], axis=1) # (N, 2F)
|
||||
wide_groups = [[c for c in g] + [c + F for c in g] for g in feature_groups]
|
||||
|
||||
def score_fn(X_in: np.ndarray) -> float:
|
||||
cd_a_in = torch.from_numpy(X_in[:, :F]).to(device)
|
||||
cd_b_in = torch.from_numpy(X_in[:, F:]).to(device)
|
||||
with torch.no_grad():
|
||||
z_cd_a = towers["cd"](cd_a_in)
|
||||
z_cd_b = towers["cd"](cd_b_in)
|
||||
z_nt_a = nt([z_img_a_cached, z_cd_a])
|
||||
z_nt_b = nt([z_img_b_cached, z_cd_b])
|
||||
z_hb = hb({"a": z_nt_a, "b": z_nt_b})
|
||||
logits = hb_head(z_hb)
|
||||
probs = TF_.softmax(logits, dim=1).cpu().numpy()
|
||||
try:
|
||||
return roc_auc_score(y, probs[:, 1])
|
||||
except Exception:
|
||||
return float("nan")
|
||||
|
||||
result = permutation_importance(
|
||||
score_fn=score_fn,
|
||||
X=X_wide,
|
||||
groups=wide_groups,
|
||||
n_permutations=n_permutations,
|
||||
seed=seed + rep_idx * 100 + fold_idx,
|
||||
feature_names=feature_names,
|
||||
)
|
||||
fold_results.append(result)
|
||||
|
||||
del towers, nt, hb, img_aux, cd_aux, hb_head
|
||||
del z_img_a_cached, z_img_b_cached
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
if not fold_results:
|
||||
print(f"[F8] {run_dir.parent.name}: no folds produced usable data; skipping rep.")
|
||||
continue
|
||||
|
||||
rep_drops_matrix = np.stack([r["mean_drop"] for r in fold_results])
|
||||
rep_mean_drops.append(rep_drops_matrix.mean(axis=0))
|
||||
rep_mean_baselines.append(float(np.mean([r["baseline"] for r in fold_results])))
|
||||
if reference_names is None:
|
||||
reference_names = fold_results[0]["feature_names"]
|
||||
print(f"[F8] {run_dir.parent.name}: baseline={rep_mean_baselines[-1]:.4f} "
|
||||
f"(n_folds={len(fold_results)})")
|
||||
|
||||
if not rep_mean_drops:
|
||||
print("[F8] make_fused_clinical_importance: no reps produced usable data; skipping.")
|
||||
return
|
||||
|
||||
rep_stack = np.stack(rep_mean_drops) # (n_reps, F)
|
||||
mean_across_reps = rep_stack.mean(axis=0)
|
||||
std_across_reps = rep_stack.std(axis=0)
|
||||
baseline_mean = float(np.mean(rep_mean_baselines))
|
||||
baseline_std = float(np.std(rep_mean_baselines))
|
||||
n_reps = rep_stack.shape[0]
|
||||
names = reference_names
|
||||
|
||||
order = np.argsort(mean_across_reps)[::-1]
|
||||
sorted_names = [names[i] for i in order]
|
||||
sorted_means = mean_across_reps[order]
|
||||
sorted_stds = std_across_reps[order]
|
||||
|
||||
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
fig, ax = plt.subplots(figsize=(7.5, max(3.5, 0.32 * len(sorted_names))))
|
||||
y_pos = np.arange(len(sorted_names))
|
||||
ax.barh(
|
||||
y_pos, sorted_means, xerr=sorted_stds,
|
||||
color="#c44e52", edgecolor="black", height=0.7,
|
||||
error_kw=dict(ecolor="#444", lw=0.8, capsize=2),
|
||||
)
|
||||
ax.set_yticks(y_pos)
|
||||
ax.set_yticklabels(sorted_names, fontsize=9)
|
||||
ax.invert_yaxis()
|
||||
ax.set_xlabel(f"Mean fused-head AUC drop on clinical shuffling (mean +/- SD across {n_reps} reps)", fontsize=10)
|
||||
ax.axvline(0, color="black", linewidth=0.7)
|
||||
ax.set_title(
|
||||
"Fused-head clinical permutation importance",
|
||||
fontsize=10, fontweight="bold",
|
||||
)
|
||||
ax.grid(axis="x", alpha=0.3, linestyle="--")
|
||||
fig.tight_layout()
|
||||
|
||||
out_png = OUT_DIR / "S8e_fused_clinical_importance.png"
|
||||
out_csv = OUT_DIR / "S8e_fused_clinical_importance.csv"
|
||||
fig.savefig(out_png, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
|
||||
pd.DataFrame({
|
||||
"feature": sorted_names,
|
||||
"mean_drop_across_reps": sorted_means,
|
||||
"std_drop_across_reps": sorted_stds,
|
||||
"baseline_auc_mean": baseline_mean,
|
||||
"baseline_auc_std": baseline_std,
|
||||
"n_reps": n_reps,
|
||||
}).to_csv(out_csv, index=False)
|
||||
|
||||
print(f"Saved: {out_png}")
|
||||
print(f"Saved: {out_csv}")
|
||||
|
||||
|
||||
def _build_v4_fold_modules(fold_idx: int, cfg: dict, data, device, run_dir: Path | None = None):
|
||||
import importlib
|
||||
import torch
|
||||
from v4.classes.v4_hypertower import build_towers
|
||||
@@ -622,7 +885,8 @@ def _build_v4_fold_modules(fold_idx: int, cfg: dict, data, device):
|
||||
stage_by_name = {s["name"]: s for s in cfg["stages"]}
|
||||
nt_cfg = stage_by_name["nt"]
|
||||
hb_cfg = stage_by_name["hb"]
|
||||
ckpt_dir = V4_CKPT_RUN / "checkpoints" / f"fold{fold_idx}"
|
||||
base = run_dir if run_dir is not None else V4_CKPT_RUN
|
||||
ckpt_dir = base / "checkpoints" / f"fold{fold_idx}"
|
||||
if not ckpt_dir.exists():
|
||||
raise FileNotFoundError(f"No v4 checkpoint directory: {ckpt_dir}")
|
||||
|
||||
@@ -782,7 +1046,6 @@ def _make_oriented_disc_detail(mean_patches: dict, examples: dict, out_path: Pat
|
||||
splits = ["correct", "incorrect"]
|
||||
|
||||
fig, axes = plt.subplots(2, 2, figsize=(8, 7.8), constrained_layout=True)
|
||||
fig.patch.set_facecolor("#f6f6f6")
|
||||
|
||||
# Column headers
|
||||
for ci, split in enumerate(splits):
|
||||
@@ -819,11 +1082,6 @@ def _make_oriented_disc_detail(mean_patches: dict, examples: dict, out_path: Pat
|
||||
ha="center", va="center", color="white", fontsize=9)
|
||||
ax.set_xticks([]); ax.set_yticks([])
|
||||
|
||||
fig.suptitle(
|
||||
"Mean image-tower Grad-CAM, disc-centred (OD-oriented)\n"
|
||||
"rows: ground truth · columns: prediction outcome · dashed circle = mean disc boundary",
|
||||
fontsize=11, fontweight="bold",
|
||||
)
|
||||
fig.savefig(out_path, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {out_path}")
|
||||
@@ -1120,6 +1378,7 @@ def main(*, run_gradcam: bool = False, n_permutations: int = 30,
|
||||
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
make_fusion_event_panel(split=fusion_split)
|
||||
make_clinical_importance(n_permutations=n_permutations)
|
||||
make_fused_clinical_importance(n_permutations=n_permutations)
|
||||
if run_gradcam:
|
||||
make_oriented_gradcam()
|
||||
else:
|
||||
|
||||
@@ -101,10 +101,6 @@ def render() -> None:
|
||||
ax.set_xticks(x)
|
||||
ax.set_xticklabels([QUAD_LABEL[q] for q in QUAD_ORDER], fontsize=11)
|
||||
ax.set_ylabel("Mean fraction of full-image Grad-CAM intensity", fontsize=11)
|
||||
ax.set_title(
|
||||
"Image-tower Grad-CAM by optic-disc quadrant (OD-oriented; centroid-split)",
|
||||
fontsize=12.5, fontweight="bold",
|
||||
)
|
||||
ax.grid(axis="y", alpha=0.3, linestyle="--")
|
||||
ax.set_ylim(0, max(ax.get_ylim()[1], 0.7))
|
||||
ax.axhline(0.25, color="#888", linestyle=":", linewidth=0.8, alpha=0.6, zorder=0)
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 137 KiB After Width: | Height: | Size: 391 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 581 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 470 KiB After Width: | Height: | Size: 445 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 335 KiB After Width: | Height: | Size: 275 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 144 KiB After Width: | Height: | Size: 122 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 101 KiB After Width: | Height: | Size: 82 KiB |
@@ -4,7 +4,6 @@ IOP_corr,0.0645588235294118,0.053595964337907635,0.7117647058823529
|
||||
Phakic/Pseudophakic,0.031004901960784353,0.05039528009700277,0.7117647058823529
|
||||
Pachymetry,0.011421568627451003,0.016547911692771797,0.7117647058823529
|
||||
Gender,0.006102941176470622,0.022596178307964714,0.7117647058823529
|
||||
eyeID,0.0,0.0,0.7117647058823529
|
||||
dioptre_2,-0.0010294117647058861,0.003622645200240852,0.7117647058823529
|
||||
astigmatism,-0.002156862745098008,0.008336072214271729,0.7117647058823529
|
||||
dioptre_1,-0.006593137254901939,0.011003619078899692,0.7117647058823529
|
||||
|
||||
|
Binary file not shown.
|
Before Width: | Height: | Size: 67 KiB After Width: | Height: | Size: 66 KiB |
@@ -0,0 +1,9 @@
|
||||
feature,mean_drop_across_reps,std_drop_across_reps,baseline_auc_mean,baseline_auc_std,n_reps
|
||||
Age,0.01915686274509804,0.005535822316487697,0.8955147058823529,0.01575126319974185,10
|
||||
IOP_corr,0.015360294117647059,0.006440372950299282,0.8955147058823529,0.01575126319974185,10
|
||||
Gender,0.010316176470588235,0.005983911392207313,0.8955147058823529,0.01575126319974185,10
|
||||
Pachymetry,0.006735294117647063,0.006456431334319213,0.8955147058823529,0.01575126319974185,10
|
||||
Phakic/Pseudophakic,0.0025857843137254933,0.005779537014891194,0.8955147058823529,0.01575126319974185,10
|
||||
astigmatism,2.6960784313724555e-05,0.0012103203246870743,0.8955147058823529,0.01575126319974185,10
|
||||
dioptre_2,-0.00013480392156862828,0.0007757846606353119,0.8955147058823529,0.01575126319974185,10
|
||||
dioptre_1,-0.0029166666666666646,0.003095754152323194,0.8955147058823529,0.01575126319974185,10
|
||||
|
Binary file not shown.
|
After Width: | Height: | Size: 59 KiB |
@@ -0,0 +1,10 @@
|
||||
feature,mean_drop_across_reps,std_drop_across_reps,baseline_auc_mean,baseline_auc_std,n_reps
|
||||
Age,0.013105392156862749,0.005436527005752407,0.8887499999999999,0.02755940814873661,10
|
||||
IOP_corr,0.008683823529411766,0.0033213280421355005,0.8887499999999999,0.02755940814873661,10
|
||||
Gender,0.007700980392156867,0.011394806861639722,0.8887499999999999,0.02755940814873661,10
|
||||
Pachymetry,0.0006838235294117676,0.004483049502641836,0.8887499999999999,0.02755940814873661,10
|
||||
dioptre_2,-0.0002941176470588276,0.0006423252286392011,0.8887499999999999,0.02755940814873661,10
|
||||
astigmatism,-0.0006887254901960787,0.0016973618769747256,0.8887499999999999,0.02755940814873661,10
|
||||
Phakic/Pseudophakic,-0.0009215686274509783,0.007695338418275053,0.8887499999999999,0.02755940814873661,10
|
||||
Axial_Length,-0.001026960784313726,0.003170425325037298,0.8887499999999999,0.02755940814873661,10
|
||||
dioptre_1,-0.003914215686274506,0.0010102113446127624,0.8887499999999999,0.02755940814873661,10
|
||||
|
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 57 KiB |
@@ -1,10 +0,0 @@
|
||||
feature,mean_importance,std_importance
|
||||
Age,0.03184101084778838,0.017362350323790826
|
||||
IOP_corr,0.0209298093602393,0.01619384191432414
|
||||
Gender,0.010245143460808846,0.020271910566046904
|
||||
Phakic/Pseudophakic,0.005718439848904106,0.019878985551523384
|
||||
eyeID,0.002907359675067821,0.0068517202428395665
|
||||
Pachymetry,0.0027483776870354105,0.008864856763204554
|
||||
dioptre_2,8.49432781653429e-05,0.0010216320594945336
|
||||
astigmatism,-0.0012830897957007016,0.0044473592790778
|
||||
dioptre_1,-0.002925119731183013,0.006220349074293031
|
||||
|
Binary file not shown.
|
Before Width: | Height: | Size: 57 KiB |
Reference in New Issue
Block a user