update 3-19
This commit is contained in:
rpotter6298
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Aggregate raw GradCAM heatmaps across all folds for a run.
|
||||
|
||||
For each combination of (eye, class, correct/incorrect) computes:
|
||||
- mean heatmap
|
||||
- std heatmap
|
||||
- count
|
||||
|
||||
Also computes a scalar per patient: fraction of GradCAM attention mass that
|
||||
falls within the expert-segmented optic disc region (from GT contour files),
|
||||
using the manifest.csv to locate the contour for each patient/eye.
|
||||
|
||||
Outputs
|
||||
-------
|
||||
{out_dir}/mean_heatmaps.npz
|
||||
Keys: {eye}_{class_name}_{correct|incorrect}_{mean|std|count}
|
||||
e.g. OD_Glaucoma_correct_mean shape (224, 224)
|
||||
|
||||
{out_dir}/attention_stats.csv
|
||||
per-patient scalars: patient_id, fold, eye, true_name, pred_name,
|
||||
correct, confidence, disc_frac, entropy
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/explainability/aggregate_gradcam.py \
|
||||
--run-dir analysis_data/pipeline_nocrop \
|
||||
--eval-mode binary \
|
||||
--tower-mode single
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
|
||||
def load_disc_mask(contour_path: Path, orig_size: tuple[int, int],
|
||||
cam_h: int, cam_w: int) -> np.ndarray | None:
|
||||
"""
|
||||
Load a PAPILA disc contour TXT file, polygon-fill at original image
|
||||
dimensions, then resize to (cam_h, cam_w). Returns a bool array or
|
||||
None if the contour cannot be loaded.
|
||||
"""
|
||||
try:
|
||||
arr = np.loadtxt(str(contour_path), dtype=np.float32)
|
||||
except Exception:
|
||||
return None
|
||||
if arr.ndim == 1:
|
||||
arr = arr.reshape(-1, 2)
|
||||
if arr.shape[0] < 3 or arr.shape[1] < 2:
|
||||
return None
|
||||
|
||||
# orig_size is (W, H) as PIL convention
|
||||
img = Image.new("L", orig_size, 0)
|
||||
draw = ImageDraw.Draw(img)
|
||||
draw.polygon([tuple(pt) for pt in arr[:, :2]], fill=1)
|
||||
mask = np.array(img.resize((cam_w, cam_h), Image.NEAREST), dtype=bool)
|
||||
return mask
|
||||
|
||||
|
||||
def build_disc_lookup(manifest_path: Path) -> dict[tuple[int, str], tuple[Path, tuple[int, int]]]:
|
||||
"""
|
||||
Returns {(patient_id_int, eye): (disc_contour_path, (img_W, img_H))}.
|
||||
Only PAPILA rows are included.
|
||||
"""
|
||||
mf = pd.read_csv(manifest_path)
|
||||
lookup: dict[tuple[int, str], tuple[Path, tuple[int, int]]] = {}
|
||||
for _, row in mf.iterrows():
|
||||
sid = str(row["sample_id"])
|
||||
if not sid.startswith("papila_RET"):
|
||||
continue
|
||||
# sample_id: papila_RET002OD or papila_RET002OS
|
||||
suffix = sid[len("papila_RET"):] # e.g. "002OD"
|
||||
eye = suffix[-2:] # "OD" or "OS"
|
||||
pid = int(suffix[:-2]) # 2
|
||||
disc_path = Path(str(row["annotation_disc"]))
|
||||
img_path = Path(str(row["image_path"]))
|
||||
if not disc_path.exists():
|
||||
continue
|
||||
# read original image size once
|
||||
try:
|
||||
with Image.open(img_path) as im:
|
||||
orig_size = im.size # (W, H)
|
||||
except Exception:
|
||||
continue
|
||||
lookup[(pid, eye)] = (disc_path, orig_size)
|
||||
return lookup
|
||||
|
||||
|
||||
def attention_entropy(cam: np.ndarray) -> float:
|
||||
flat = cam.flatten().astype(np.float64)
|
||||
flat = flat / (flat.sum() + 1e-12)
|
||||
return float(-np.sum(flat * np.log(flat + 1e-12)))
|
||||
|
||||
|
||||
def load_fold(gradcam_dir: Path):
|
||||
idx_path = gradcam_dir / "gradcam_index.csv"
|
||||
if not idx_path.exists():
|
||||
return None
|
||||
idx = pd.read_csv(idx_path)
|
||||
records = []
|
||||
for _, row in idx.iterrows():
|
||||
pid = row["patient_id"]
|
||||
for eye in ("OD", "OS"):
|
||||
npy = gradcam_dir / f"patient_{pid}_{eye}_cam.npy"
|
||||
if not npy.exists():
|
||||
continue
|
||||
cam = np.load(npy)
|
||||
records.append({
|
||||
"patient_id": pid,
|
||||
"eye": eye,
|
||||
"true_label": int(row["true_label"]),
|
||||
"true_name": row["true_name"],
|
||||
"pred_label": int(row["pred_label"]),
|
||||
"pred_name": row["pred_name"],
|
||||
"confidence": float(row["confidence"]),
|
||||
"correct": bool(row["correct"]),
|
||||
"cam": cam,
|
||||
})
|
||||
return records
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--run-dir", default="analysis_data/pipeline_nocrop")
|
||||
ap.add_argument("--eval-mode", default="binary")
|
||||
ap.add_argument("--tower-mode", default="single")
|
||||
ap.add_argument("--manifest", default="manifest.csv")
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
run_dir = Path(args.run_dir)
|
||||
mode_dir = run_dir / args.eval_mode / args.tower_mode
|
||||
out_dir = mode_dir / "gradcam_aggregate"
|
||||
if args.out:
|
||||
out_dir = Path(args.out)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ---- build disc mask lookup ----
|
||||
manifest_path = Path(args.manifest)
|
||||
disc_lookup = build_disc_lookup(manifest_path)
|
||||
print(f"Disc mask lookup: {len(disc_lookup)} entries from {manifest_path}")
|
||||
|
||||
# ---- collect all records ----
|
||||
all_records = []
|
||||
stat_rows = []
|
||||
fold_dirs = sorted(
|
||||
[d for d in mode_dir.iterdir() if d.is_dir() and d.name.startswith("fold")],
|
||||
key=lambda p: int(p.name.replace("fold", "")),
|
||||
)
|
||||
if not fold_dirs:
|
||||
print(f"No fold dirs found under {mode_dir}")
|
||||
return
|
||||
|
||||
for fd in fold_dirs:
|
||||
gcam_dir = fd / "explainability" / "gradcam"
|
||||
records = load_fold(gcam_dir)
|
||||
if records is None:
|
||||
print(f" [skip] {fd.name}: no gradcam_index.csv")
|
||||
continue
|
||||
print(f" {fd.name}: {len(records)} eye records")
|
||||
for r in records:
|
||||
r["fold"] = fd.name
|
||||
all_records.append(r)
|
||||
|
||||
if not all_records:
|
||||
print("No records found — re-run explain_fold.py first.")
|
||||
return
|
||||
|
||||
print(f"\nTotal eye records: {len(all_records)}")
|
||||
|
||||
h, w = all_records[0]["cam"].shape
|
||||
|
||||
# ---- per-record stats ----
|
||||
n_missing = 0
|
||||
for r in all_records:
|
||||
cam = r["cam"]
|
||||
total = cam.sum() + 1e-12
|
||||
pid = int(r["patient_id"])
|
||||
eye = r["eye"]
|
||||
|
||||
disc_mask = None
|
||||
key = (pid, eye)
|
||||
if key in disc_lookup:
|
||||
disc_path, orig_size = disc_lookup[key]
|
||||
disc_mask = load_disc_mask(disc_path, orig_size, h, w)
|
||||
if disc_mask is None:
|
||||
n_missing += 1
|
||||
disc_frac = float("nan")
|
||||
else:
|
||||
disc_frac = float(cam[disc_mask].sum() / total)
|
||||
|
||||
stat_rows.append({
|
||||
"patient_id": r["patient_id"],
|
||||
"fold": r["fold"],
|
||||
"eye": r["eye"],
|
||||
"true_name": r["true_name"],
|
||||
"pred_name": r["pred_name"],
|
||||
"correct": r["correct"],
|
||||
"confidence": r["confidence"],
|
||||
"disc_frac": disc_frac,
|
||||
"entropy": attention_entropy(cam),
|
||||
})
|
||||
|
||||
if n_missing:
|
||||
print(f" Warning: {n_missing} records had no disc mask (disc_frac=NaN)")
|
||||
|
||||
stats_df = pd.DataFrame(stat_rows)
|
||||
stats_path = out_dir / "attention_stats.csv"
|
||||
stats_df.to_csv(stats_path, index=False)
|
||||
print(f"Saved attention stats → {stats_path}")
|
||||
|
||||
# ---- mean heatmaps ----
|
||||
npz_arrays = {}
|
||||
groups: dict[tuple, list[np.ndarray]] = {}
|
||||
for r in all_records:
|
||||
key = (r["eye"], r["true_name"], "correct" if r["correct"] else "incorrect")
|
||||
groups.setdefault(key, []).append(r["cam"])
|
||||
for r in all_records:
|
||||
key = (r["eye"], r["true_name"], "all")
|
||||
groups.setdefault(key, []).append(r["cam"])
|
||||
|
||||
for (eye, cls, split), cams in groups.items():
|
||||
stack = np.stack(cams, axis=0)
|
||||
key_base = f"{eye}_{cls}_{split}"
|
||||
npz_arrays[f"{key_base}_mean"] = stack.mean(axis=0).astype(np.float32)
|
||||
npz_arrays[f"{key_base}_std"] = stack.std(axis=0).astype(np.float32)
|
||||
npz_arrays[f"{key_base}_count"] = np.array(len(cams))
|
||||
print(f" {key_base}: N={len(cams)}")
|
||||
|
||||
npz_path = out_dir / "mean_heatmaps.npz"
|
||||
np.savez_compressed(npz_path, **npz_arrays)
|
||||
print(f"Saved mean heatmaps → {npz_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -409,6 +409,7 @@ def run_gradcam(
|
||||
overlay_grid_items: list[
|
||||
tuple[Image.Image | None, Image.Image | None, str, bool]
|
||||
] = []
|
||||
index_rows: list[dict] = []
|
||||
|
||||
model.eval()
|
||||
for batch in loader:
|
||||
@@ -492,6 +493,19 @@ def run_gradcam(
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# Save raw CAM arrays
|
||||
np.save(gradcam_dir / f"patient_{pid}_OD_cam.npy", cam_od)
|
||||
np.save(gradcam_dir / f"patient_{pid}_OS_cam.npy", cam_os)
|
||||
index_rows.append({
|
||||
"patient_id": pid,
|
||||
"true_label": label,
|
||||
"true_name": true_name,
|
||||
"pred_label": pred,
|
||||
"pred_name": pred_name,
|
||||
"confidence": conf,
|
||||
"correct": correct,
|
||||
})
|
||||
|
||||
# Accumulate for summary grid
|
||||
od_overlay = overlay_gradcam(orig_od, cam_od, alpha) if orig_od else None
|
||||
os_overlay = overlay_gradcam(orig_os, cam_os, alpha) if orig_os else None
|
||||
@@ -500,6 +514,16 @@ def run_gradcam(
|
||||
|
||||
gcam.remove()
|
||||
|
||||
# ---- save index CSV ----
|
||||
if index_rows:
|
||||
import csv
|
||||
idx_path = gradcam_dir / "gradcam_index.csv"
|
||||
with idx_path.open("w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=list(index_rows[0].keys()))
|
||||
writer.writeheader()
|
||||
writer.writerows(index_rows)
|
||||
print(f" Index CSV → {idx_path}", flush=True)
|
||||
|
||||
# ---- summary grid: N_patients rows × 2 cols (OD overlay | OS overlay) ----
|
||||
n = len(overlay_grid_items)
|
||||
if n == 0:
|
||||
|
||||
@@ -362,94 +362,114 @@ def run_gradcam(
|
||||
gradcam_dir = out_dir / "gradcam"
|
||||
gradcam_dir.mkdir(exist_ok=True)
|
||||
|
||||
target_layer = get_gradcam_layer(model, backbone)
|
||||
gcam = GradCAM(target_layer)
|
||||
|
||||
num_classes = model.bridge.classifier_fused[-1].out_features
|
||||
manifest_path = gradcam_dir / "gradcam_manifest.csv"
|
||||
skip_overlays = manifest_path.exists()
|
||||
overlay_grid_items = []
|
||||
|
||||
model.eval()
|
||||
for batch in loader:
|
||||
img_od = batch["image_1"].to(device)
|
||||
img_os = batch["image_2"].to(device)
|
||||
meta_od = batch["matrix_1"].to(device)
|
||||
meta_os = batch["matrix_2"].to(device)
|
||||
lbl_raw = batch["label_1"][0]
|
||||
label = int(lbl_raw.item() if isinstance(lbl_raw, torch.Tensor) else lbl_raw)
|
||||
pid = batch["id_1"][0]
|
||||
if skip_overlays:
|
||||
print(" Overlays already exist — skipping computation, loading from disk.", flush=True)
|
||||
manifest_df = pd.read_csv(manifest_path)
|
||||
for _, row in manifest_df.iterrows():
|
||||
pid = str(row["pid"])
|
||||
od_path = gradcam_dir / f"gradcam_od_{pid}.png"
|
||||
os_path = gradcam_dir / f"gradcam_os_{pid}.png"
|
||||
od_ov = Image.open(od_path).convert("RGB") if od_path.exists() else None
|
||||
os_ov = Image.open(os_path).convert("RGB") if os_path.exists() else None
|
||||
overlay_grid_items.append((od_ov, os_ov, str(row["short_lbl"]), bool(row["correct"])))
|
||||
else:
|
||||
target_layer = get_gradcam_layer(model, backbone)
|
||||
gcam = GradCAM(target_layer)
|
||||
manifest_rows = []
|
||||
|
||||
cam_od, pred = gcam.compute(img_od, meta_od, model)
|
||||
cam_os, _ = gcam.compute(img_os, meta_os, model)
|
||||
model.eval()
|
||||
for batch in loader:
|
||||
img_od = batch["image_1"].to(device)
|
||||
img_os = batch["image_2"].to(device)
|
||||
meta_od = batch["matrix_1"].to(device)
|
||||
meta_os = batch["matrix_2"].to(device)
|
||||
lbl_raw = batch["label_1"][0]
|
||||
label = int(lbl_raw.item() if isinstance(lbl_raw, torch.Tensor) else lbl_raw)
|
||||
pid = batch["id_1"][0]
|
||||
|
||||
with torch.no_grad():
|
||||
out_od = model(img_od, meta_od)
|
||||
conf = float(torch.softmax(out_od, dim=1)[0, pred].item())
|
||||
cam_od, pred = gcam.compute(img_od, meta_od, model)
|
||||
cam_os, _ = gcam.compute(img_os, meta_os, model)
|
||||
|
||||
row_od = eval_df[
|
||||
(eval_df["Patient ID"] == int(pid)) & (eval_df["eyeID"] == "OD")
|
||||
]
|
||||
row_os = eval_df[
|
||||
(eval_df["Patient ID"] == int(pid)) & (eval_df["eyeID"] == "OS")
|
||||
]
|
||||
orig_od = (
|
||||
Image.open(data.get_image_path(row_od.iloc[0])).convert("RGB")
|
||||
if len(row_od) else None
|
||||
)
|
||||
orig_os = (
|
||||
Image.open(data.get_image_path(row_os.iloc[0])).convert("RGB")
|
||||
if len(row_os) else None
|
||||
)
|
||||
with torch.no_grad():
|
||||
out_od = model(img_od, meta_od)
|
||||
conf = float(torch.softmax(out_od, dim=1)[0, pred].item())
|
||||
|
||||
true_name = label_name(label, eval_mode)
|
||||
pred_name = label_name(pred, eval_mode)
|
||||
correct = label == pred
|
||||
title = (
|
||||
f"Patient {pid} | True: {true_name} | Pred: {pred_name} "
|
||||
f"| conf={conf:.2f} {'✓' if correct else '✗'}"
|
||||
)
|
||||
row_od = eval_df[
|
||||
(eval_df["Patient ID"] == int(pid)) & (eval_df["eyeID"] == "OD")
|
||||
]
|
||||
row_os = eval_df[
|
||||
(eval_df["Patient ID"] == int(pid)) & (eval_df["eyeID"] == "OS")
|
||||
]
|
||||
orig_od = (
|
||||
Image.open(data.get_image_path(row_od.iloc[0])).convert("RGB")
|
||||
if len(row_od) else None
|
||||
)
|
||||
orig_os = (
|
||||
Image.open(data.get_image_path(row_os.iloc[0])).convert("RGB")
|
||||
if len(row_os) else None
|
||||
)
|
||||
|
||||
fig, axes = plt.subplots(2, 2, figsize=(10, 9))
|
||||
fig.suptitle(title, fontsize=11, fontweight="bold", color="green" if correct else "red")
|
||||
true_name = label_name(label, eval_mode)
|
||||
pred_name = label_name(pred, eval_mode)
|
||||
correct = label == pred
|
||||
title = (
|
||||
f"Patient {pid} | True: {true_name} | Pred: {pred_name} "
|
||||
f"| conf={conf:.2f} {'✓' if correct else '✗'}"
|
||||
)
|
||||
|
||||
if orig_od is not None:
|
||||
axes[0, 0].imshow(orig_od)
|
||||
axes[0, 0].set_title("OD — original", fontsize=9)
|
||||
axes[0, 1].imshow(overlay_gradcam(orig_od, cam_od, alpha))
|
||||
axes[0, 1].set_title("OD — GradCAM", fontsize=9)
|
||||
else:
|
||||
axes[0, 0].set_title("OD — (missing)", fontsize=9)
|
||||
axes[0, 0].axis("off")
|
||||
axes[0, 1].axis("off")
|
||||
fig, axes = plt.subplots(2, 2, figsize=(10, 9))
|
||||
fig.suptitle(title, fontsize=11, fontweight="bold", color="green" if correct else "red")
|
||||
|
||||
if orig_os is not None:
|
||||
axes[1, 0].imshow(orig_os)
|
||||
axes[1, 0].set_title("OS — original", fontsize=9)
|
||||
axes[1, 1].imshow(overlay_gradcam(orig_os, cam_os, alpha))
|
||||
axes[1, 1].set_title("OS — GradCAM", fontsize=9)
|
||||
else:
|
||||
axes[1, 0].set_title("OS — (missing)", fontsize=9)
|
||||
axes[1, 0].axis("off")
|
||||
axes[1, 1].axis("off")
|
||||
if orig_od is not None:
|
||||
axes[0, 0].imshow(orig_od)
|
||||
axes[0, 0].set_title("OD — original", fontsize=9)
|
||||
axes[0, 1].imshow(overlay_gradcam(orig_od, cam_od, alpha))
|
||||
axes[0, 1].set_title("OD — GradCAM", fontsize=9)
|
||||
else:
|
||||
axes[0, 0].set_title("OD — (missing)", fontsize=9)
|
||||
axes[0, 0].axis("off")
|
||||
axes[0, 1].axis("off")
|
||||
|
||||
fig.tight_layout()
|
||||
out_path = gradcam_dir / f"patient_{pid}_OD_OS.png"
|
||||
fig.savefig(out_path, dpi=120)
|
||||
plt.close(fig)
|
||||
print(f" Patient {pid}: {true_name} → {pred_name} ({conf:.2f}) → {out_path.name}", flush=True)
|
||||
if orig_os is not None:
|
||||
axes[1, 0].imshow(orig_os)
|
||||
axes[1, 0].set_title("OS — original", fontsize=9)
|
||||
axes[1, 1].imshow(overlay_gradcam(orig_os, cam_os, alpha))
|
||||
axes[1, 1].set_title("OS — GradCAM", fontsize=9)
|
||||
else:
|
||||
axes[1, 0].set_title("OS — (missing)", fontsize=9)
|
||||
axes[1, 0].axis("off")
|
||||
axes[1, 1].axis("off")
|
||||
|
||||
od_overlay = overlay_gradcam(orig_od, cam_od, alpha) if orig_od else None
|
||||
os_overlay = overlay_gradcam(orig_os, cam_os, alpha) if orig_os else None
|
||||
short_lbl = f"P{pid} {true_name[:3]}→{pred_name[:3]} {'✓' if correct else '✗'}"
|
||||
overlay_grid_items.append((od_overlay, os_overlay, short_lbl, correct))
|
||||
fig.tight_layout()
|
||||
out_path = gradcam_dir / f"patient_{pid}_OD_OS.png"
|
||||
fig.savefig(out_path, dpi=120)
|
||||
plt.close(fig)
|
||||
print(f" Patient {pid}: {true_name} → {pred_name} ({conf:.2f}) → {out_path.name}", flush=True)
|
||||
|
||||
gcam.remove()
|
||||
od_overlay = overlay_gradcam(orig_od, cam_od, alpha) if orig_od else None
|
||||
os_overlay = overlay_gradcam(orig_os, cam_os, alpha) if orig_os else None
|
||||
if od_overlay is not None:
|
||||
od_overlay.save(gradcam_dir / f"gradcam_od_{pid}.png")
|
||||
if os_overlay is not None:
|
||||
os_overlay.save(gradcam_dir / f"gradcam_os_{pid}.png")
|
||||
short_lbl = f"P{pid} {true_name[:3]}→{pred_name[:3]} {'✓' if correct else '✗'}"
|
||||
overlay_grid_items.append((od_overlay, os_overlay, short_lbl, correct))
|
||||
manifest_rows.append({"pid": pid, "short_lbl": short_lbl, "correct": correct})
|
||||
|
||||
gcam.remove()
|
||||
pd.DataFrame(manifest_rows).to_csv(manifest_path, index=False)
|
||||
|
||||
# Always regenerate the summary grid
|
||||
n = len(overlay_grid_items)
|
||||
if n == 0:
|
||||
print(" [Phase 2] No patients to visualise.", flush=True)
|
||||
return
|
||||
|
||||
fig, axes = plt.subplots(n, 2, figsize=(8, n * 3.2 + 0.8))
|
||||
fig, axes = plt.subplots(n, 2, figsize=(8, n * 3.2 + 1.5))
|
||||
if n == 1:
|
||||
axes = axes[np.newaxis, :]
|
||||
fig.suptitle("GradCAM Summary Grid — all holdout patients", fontsize=12)
|
||||
@@ -465,7 +485,7 @@ def run_gradcam(
|
||||
axes[i, 1].imshow(os_ov)
|
||||
axes[i, 1].set_title(f"{lbl}\nOS", fontsize=7, color=color)
|
||||
|
||||
fig.tight_layout()
|
||||
fig.tight_layout(rect=[0, 0, 1, 0.97])
|
||||
grid_path = out_dir / "gradcam_summary_grid.png"
|
||||
fig.savefig(grid_path, dpi=120)
|
||||
plt.close(fig)
|
||||
@@ -496,6 +516,7 @@ def _fusion_event_stats(
|
||||
pm: np.ndarray,
|
||||
split_name: str,
|
||||
out_dir: Path,
|
||||
component_labels: tuple[str, str] = ("img", "md"),
|
||||
) -> dict:
|
||||
"""Compute, save, and plot fusion events for one split. Returns summary dict."""
|
||||
N = len(y_true)
|
||||
@@ -503,6 +524,16 @@ def _fusion_event_stats(
|
||||
print(f" [{split_name}] No samples — skipping.", flush=True)
|
||||
return {}
|
||||
|
||||
a, b = component_labels
|
||||
event_labels = [
|
||||
"full correction\n(both wrong→fused right)",
|
||||
f"{a} assist\n({a} wrong, {b} right→right)",
|
||||
f"{b} assist\n({b} wrong, {a} right→right)",
|
||||
"full error\n(both right→fused wrong)",
|
||||
f"{a} drag\n({a} wrong, {b} right→wrong)",
|
||||
f"{b} drag\n({b} wrong, {a} right→wrong)",
|
||||
]
|
||||
|
||||
pred_f = pf.argmax(axis=1)
|
||||
pred_i = pi.argmax(axis=1)
|
||||
pred_m = pm.argmax(axis=1)
|
||||
@@ -529,7 +560,7 @@ def _fusion_event_stats(
|
||||
counts = [int(m.sum()) for m in event_masks]
|
||||
|
||||
print(f"\n [{split_name}] N={N}", flush=True)
|
||||
for label, count in zip(_EVENT_LABELS, counts):
|
||||
for label, count in zip(event_labels, counts):
|
||||
print(f" {label.replace(chr(10), ' '):55s}: {count}", flush=True)
|
||||
n_corr, n_err = counts[0], counts[3]
|
||||
print(f" full correction/error ratio: {n_corr}/{n_err}", flush=True)
|
||||
@@ -582,11 +613,11 @@ def _fusion_event_stats(
|
||||
axes[0].set_xticklabels(["Positive\nevents", "Negative\nevents"])
|
||||
axes[0].set_ylabel("Count")
|
||||
patches = [mpatches.Patch(color=c, label=l.replace("\n", " "))
|
||||
for c, l in zip(_EVENT_COLORS, _EVENT_LABELS)]
|
||||
for c, l in zip(_EVENT_COLORS, event_labels)]
|
||||
axes[0].legend(handles=patches, fontsize=6, loc="upper right")
|
||||
|
||||
box_data = [conf_delta[m] for m in event_masks if m.sum() > 0]
|
||||
box_labels = [l.split("\n")[0] for m, l in zip(event_masks, _EVENT_LABELS) if m.sum() > 0]
|
||||
box_labels = [l.split("\n")[0] for m, l in zip(event_masks, event_labels) if m.sum() > 0]
|
||||
box_cols = [c for m, c in zip(event_masks, _EVENT_COLORS) if m.sum() > 0]
|
||||
if box_data:
|
||||
bp = axes[1].boxplot(box_data, patch_artist=True, widths=0.5)
|
||||
@@ -595,10 +626,10 @@ def _fusion_event_stats(
|
||||
axes[1].set_xticks(range(1, len(box_labels) + 1))
|
||||
axes[1].set_xticklabels(box_labels, rotation=35, ha="right", fontsize=7)
|
||||
axes[1].axhline(0, color="black", linewidth=0.8, linestyle="--")
|
||||
axes[1].set_ylabel("conf_delta\n(fused − avg(img, md))")
|
||||
axes[1].set_ylabel(f"conf_delta\n(fused − avg({a}, {b}))")
|
||||
axes[1].set_title("Confidence delta by event type")
|
||||
|
||||
for mask, color, label in zip(event_masks, _EVENT_COLORS, _EVENT_LABELS):
|
||||
for mask, color, label in zip(event_masks, _EVENT_COLORS, event_labels):
|
||||
if mask.sum() > 0:
|
||||
axes[2].scatter(conf_i[mask], conf_m[mask], c=color,
|
||||
label=label.split("\n")[0], alpha=0.85, s=45, edgecolors="none")
|
||||
@@ -609,8 +640,8 @@ def _fusion_event_stats(
|
||||
axes[2].scatter(conf_i[concordant_bad], conf_m[concordant_bad],
|
||||
c="darkgrey", alpha=0.4, s=20, edgecolors="none", label="concordant wrong")
|
||||
axes[2].plot([0, 1], [0, 1], "k--", linewidth=0.5, alpha=0.4)
|
||||
axes[2].set_xlabel("conf_img")
|
||||
axes[2].set_ylabel("conf_md")
|
||||
axes[2].set_xlabel(f"conf_{a}")
|
||||
axes[2].set_ylabel(f"conf_{b}")
|
||||
axes[2].set_title("Tower confidence space\ncoloured by fusion event")
|
||||
axes[2].legend(fontsize=6, loc="lower right")
|
||||
|
||||
@@ -711,12 +742,13 @@ def run_fusion_event_analysis(fold_dir: Path, out_dir: Path) -> list[dict]:
|
||||
pm_ens = 0.5 * (pm_od + pm_os)
|
||||
summaries.append(_fusion_event_stats(y_val, pf_ens, pi_ens, pm_ens, "val", out_dir))
|
||||
|
||||
# Fused head (if available): learned bilateral combination vs averaged towers
|
||||
# Fused head (if available): learned bilateral combination vs per-eye bridge outputs
|
||||
fused_head_f = fold_dir / "probs_fused_head.npy"
|
||||
if fused_head_f.exists():
|
||||
pf_head = np.load(fused_head_f)
|
||||
summaries.append(_fusion_event_stats(
|
||||
y_val, pf_head, pi_ens, pm_ens, "val_fused_head", out_dir))
|
||||
y_val, pf_head, pf_od, pf_os, "val_fused_head", out_dir,
|
||||
component_labels=("OD", "OS")))
|
||||
else:
|
||||
print(" Val epoch files not found — skipping val.", flush=True)
|
||||
|
||||
@@ -745,6 +777,95 @@ def run_fusion_event_analysis(fold_dir: Path, out_dir: Path) -> list[dict]:
|
||||
return summaries
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cross-fold MD importance summary plot
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _plot_run_md_importance_summary(run_dir: Path, folds: list) -> None:
|
||||
"""Aggregate per-fold MD permutation importance CSVs into a run-level summary plot."""
|
||||
all_dfs = []
|
||||
for fold_idx, fold_dir, _ in folds:
|
||||
csv_path = fold_dir / "explainability" / "md_permutation_importance.csv"
|
||||
if csv_path.exists():
|
||||
df = pd.read_csv(csv_path)
|
||||
df["fold"] = fold_idx
|
||||
all_dfs.append(df)
|
||||
|
||||
if not all_dfs:
|
||||
print(" [MD summary] No per-fold importance CSVs found — skipping.", flush=True)
|
||||
return
|
||||
|
||||
combined = pd.concat(all_dfs, ignore_index=True)
|
||||
_SPECIAL = {"TOTAL_MD_ABLATION", "GAUSSIAN_NOISE_ABLATION"}
|
||||
feature_rows = combined[~combined["feature"].isin(_SPECIAL)]
|
||||
special_rows = combined[combined["feature"].isin(_SPECIAL)]
|
||||
|
||||
agg = (
|
||||
feature_rows.groupby("feature")["importance"]
|
||||
.agg(["mean", "std"])
|
||||
.reset_index()
|
||||
.rename(columns={"mean": "mean_importance", "std": "std_importance"})
|
||||
.sort_values("mean_importance", ascending=False)
|
||||
.reset_index(drop=True)
|
||||
)
|
||||
special_agg = (
|
||||
special_rows.groupby("feature")["importance"]
|
||||
.agg(["mean", "std"])
|
||||
.reset_index()
|
||||
)
|
||||
|
||||
names = agg["feature"].tolist()
|
||||
imps = agg["mean_importance"].tolist()
|
||||
stds = agg["std_importance"].fillna(0).tolist()
|
||||
colors = ["#e05c5c" if v >= 0 else "#5c9ee0" for v in imps]
|
||||
|
||||
fig, ax = plt.subplots(figsize=(9, max(4, (len(names) + 3) * 0.45)))
|
||||
y_pos = np.arange(len(names))
|
||||
ax.barh(y_pos, imps, xerr=stds, color=colors, ecolor="grey", capsize=3, height=0.6)
|
||||
ax.axhline(len(names) - 0.25, color="grey", linewidth=0.6, linestyle="--")
|
||||
|
||||
special_label_map = {
|
||||
"TOTAL_MD_ABLATION": "ALL MD (permute)",
|
||||
"GAUSSIAN_NOISE_ABLATION": "ALL MD (noise)",
|
||||
}
|
||||
special_colors = {
|
||||
"TOTAL_MD_ABLATION": "#c45ce0",
|
||||
"GAUSSIAN_NOISE_ABLATION": "#e08c2a",
|
||||
}
|
||||
extra_ytick_pos = []
|
||||
extra_ytick_labels = []
|
||||
for i, feat in enumerate(["TOTAL_MD_ABLATION", "GAUSSIAN_NOISE_ABLATION"]):
|
||||
row = special_agg[special_agg["feature"] == feat]
|
||||
if row.empty:
|
||||
continue
|
||||
offset = len(names) + 0.5 + i
|
||||
val, err = float(row["mean"].iloc[0]), float(row["std"].iloc[0])
|
||||
ax.barh(offset, val, xerr=err,
|
||||
color=special_colors[feat] if val >= 0 else "#5c9ee0",
|
||||
ecolor="grey", capsize=3, height=0.6)
|
||||
extra_ytick_pos.append(offset)
|
||||
extra_ytick_labels.append(special_label_map[feat])
|
||||
|
||||
ax.set_yticks(list(y_pos) + extra_ytick_pos)
|
||||
ax.set_yticklabels(names + extra_ytick_labels, fontsize=9)
|
||||
ax.invert_yaxis()
|
||||
ax.axvline(0, color="black", linewidth=0.8)
|
||||
ax.set_xlabel("Mean AUC drop (baseline − permuted)", fontsize=10)
|
||||
ax.set_title(
|
||||
f"MD Tower — Permutation Feature Importance ({len(all_dfs)}-fold summary)\n"
|
||||
f"error bars = std across folds",
|
||||
fontsize=11,
|
||||
)
|
||||
fig.tight_layout()
|
||||
out_path = run_dir / "explainability_md_importance_summary.png"
|
||||
fig.savefig(out_path, dpi=150)
|
||||
plt.close(fig)
|
||||
print(f" MD importance summary → {out_path}", flush=True)
|
||||
|
||||
agg.to_csv(run_dir / "explainability_md_importance_summary.csv", index=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cross-fold fusion summary plot
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -761,6 +882,17 @@ def _plot_cross_fold_fusion_summary(df_sum: pd.DataFrame, run_dir: Path) -> None
|
||||
n_folds = len(grp)
|
||||
fold_ids = grp["fold"].values
|
||||
|
||||
# For the fused_head split the two comparators are OD/OS bridges, not img/md towers
|
||||
comp_a, comp_b = ("OD", "OS") if "fused_head" in split_name else ("img", "md")
|
||||
summary_event_labels = [
|
||||
"full correction\n(both wrong→fused right)",
|
||||
f"{comp_a} assist\n({comp_a} wrong, {comp_b} right→right)",
|
||||
f"{comp_b} assist\n({comp_b} wrong, {comp_a} right→right)",
|
||||
"full error\n(both right→fused wrong)",
|
||||
f"{comp_a} drag\n({comp_a} wrong, {comp_b} right→wrong)",
|
||||
f"{comp_b} drag\n({comp_b} wrong, {comp_a} right→wrong)",
|
||||
]
|
||||
|
||||
# Load all per-fold CSVs for this split to get sample-level data
|
||||
sample_dfs = []
|
||||
for fold_idx in fold_ids:
|
||||
@@ -790,7 +922,7 @@ def _plot_cross_fold_fusion_summary(df_sum: pd.DataFrame, run_dir: Path) -> None
|
||||
axes[0].set_xticklabels(["Positive\nevents", "Negative\nevents"])
|
||||
axes[0].set_ylabel("Count (all folds)")
|
||||
patches = [mpatches.Patch(color=c, label=l.replace("\n", " "))
|
||||
for c, l in zip(_EVENT_COLORS, _EVENT_LABELS)]
|
||||
for c, l in zip(_EVENT_COLORS, summary_event_labels)]
|
||||
axes[0].legend(handles=patches, fontsize=6, loc="upper right")
|
||||
|
||||
# Panel 2: per-fold stacked bar (fold variance)
|
||||
@@ -819,7 +951,7 @@ def _plot_cross_fold_fusion_summary(df_sum: pd.DataFrame, run_dir: Path) -> None
|
||||
axes[2].axhline(0, color="grey", linewidth=0.7)
|
||||
axes[2].set_xticks(x)
|
||||
axes[2].set_xticklabels([f"fold {f}" for f in fold_ids], fontsize=8)
|
||||
axes[2].set_ylabel("conf_delta mean\n(fused − avg(img, md))")
|
||||
axes[2].set_ylabel(f"conf_delta mean\n(fused − avg({comp_a}, {comp_b}))")
|
||||
axes[2].set_title("Confidence delta per fold")
|
||||
axes[2].legend(fontsize=8)
|
||||
|
||||
@@ -829,7 +961,7 @@ def _plot_cross_fold_fusion_summary(df_sum: pd.DataFrame, run_dir: Path) -> None
|
||||
box_data = [sample_df.loc[sample_df["event_type"] == k, "conf_delta"].values
|
||||
for k in key_order]
|
||||
box_labels = [l.split("\n")[0]
|
||||
for k, l in zip(_EVENT_KEYS, _EVENT_LABELS) if k in key_order]
|
||||
for k, l in zip(_EVENT_KEYS, summary_event_labels) if k in key_order]
|
||||
box_cols = [c for k, c in zip(_EVENT_KEYS, _EVENT_COLORS) if k in key_order]
|
||||
if box_data:
|
||||
bp = axes[3].boxplot(box_data, patch_artist=True, widths=0.5)
|
||||
@@ -838,7 +970,7 @@ def _plot_cross_fold_fusion_summary(df_sum: pd.DataFrame, run_dir: Path) -> None
|
||||
axes[3].set_xticks(range(1, len(box_labels) + 1))
|
||||
axes[3].set_xticklabels(box_labels, rotation=35, ha="right", fontsize=7)
|
||||
axes[3].axhline(0, color="black", linewidth=0.8, linestyle="--")
|
||||
axes[3].set_ylabel("conf_delta\n(fused − avg(img, md))")
|
||||
axes[3].set_ylabel(f"conf_delta\n(fused − avg({comp_a}, {comp_b}))")
|
||||
axes[3].set_title("Confidence delta by event type\n(all folds)")
|
||||
|
||||
# Panel 5: tower confidence space scatter (all folds combined)
|
||||
@@ -847,7 +979,7 @@ def _plot_cross_fold_fusion_summary(df_sum: pd.DataFrame, run_dir: Path) -> None
|
||||
for key, color in zip(_EVENT_KEYS, _EVENT_COLORS):
|
||||
sub = sample_df[sample_df["event_type"] == key]
|
||||
if len(sub):
|
||||
label = next(l.split("\n")[0] for k, l in zip(_EVENT_KEYS, _EVENT_LABELS)
|
||||
label = next(l.split("\n")[0] for k, l in zip(_EVENT_KEYS, summary_event_labels)
|
||||
if k == key)
|
||||
axes[4].scatter(sub["conf_img"], sub["conf_md"], c=color,
|
||||
label=label, alpha=0.7, s=30, edgecolors="none")
|
||||
@@ -860,8 +992,8 @@ def _plot_cross_fold_fusion_summary(df_sum: pd.DataFrame, run_dir: Path) -> None
|
||||
axes[4].scatter(sub["conf_img"], sub["conf_md"], c=conc_color,
|
||||
alpha=0.3, s=15, edgecolors="none", label=conc_label)
|
||||
axes[4].plot([0, 1], [0, 1], "k--", linewidth=0.5, alpha=0.4)
|
||||
axes[4].set_xlabel("conf_img")
|
||||
axes[4].set_ylabel("conf_md")
|
||||
axes[4].set_xlabel(f"conf_{comp_a}")
|
||||
axes[4].set_ylabel(f"conf_{comp_b}")
|
||||
axes[4].legend(fontsize=6, loc="lower right")
|
||||
axes[4].set_title("Tower confidence space\n(all folds)")
|
||||
|
||||
@@ -1105,6 +1237,12 @@ def main():
|
||||
del model
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
# ---- Cross-fold MD importance summary ----
|
||||
if not args.no_phase1:
|
||||
print(f"\n{'='*60}", flush=True)
|
||||
print("[explain_run] === Cross-fold MD importance summary ===", flush=True)
|
||||
_plot_run_md_importance_summary(run_dir, folds)
|
||||
|
||||
# ---- Cross-fold Phase 3 summary ----
|
||||
if all_phase3_summaries and not args.no_phase3:
|
||||
print(f"\n{'='*60}", flush=True)
|
||||
|
||||
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Granular disc-attention visualisation.
|
||||
|
||||
Layout (3 rows × N_class cols):
|
||||
Row 0 — disc-centred mean GradCAM patch for CORRECT predictions
|
||||
Row 1 — disc-centred mean GradCAM patch for INCORRECT predictions
|
||||
Row 2 — per-patient strip plot of disc_frac (blue=correct, red=incorrect)
|
||||
|
||||
Disc-centred patches: each patient's CAM is translated and scaled so the GT
|
||||
disc centroid sits at the patch centre before averaging. A dashed white circle
|
||||
marks the average GT disc size. This makes cross-patient averaging meaningful
|
||||
regardless of where the disc sits in the original image.
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/explainability/plot_disc_attention_detail.py \
|
||||
--agg-dir analysis_data/pipeline_nocrop/binary/single/gradcam_aggregate \
|
||||
--manifest manifest.csv
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.patches import Circle
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
|
||||
DISC_SPAN = 5 # patch side = DISC_SPAN × disc diameter
|
||||
OUTPUT_SIZE = 96 # pixel size of each thumbnail
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Disc mask helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_disc_mask(contour_path: Path, orig_size: tuple, cam_h: int, cam_w: int):
|
||||
try:
|
||||
arr = np.loadtxt(str(contour_path), dtype=np.float32)
|
||||
except Exception:
|
||||
return None
|
||||
if arr.ndim == 1:
|
||||
arr = arr.reshape(-1, 2)
|
||||
if arr.shape[0] < 3 or arr.shape[1] < 2:
|
||||
return None
|
||||
img = Image.new("L", orig_size, 0)
|
||||
ImageDraw.Draw(img).polygon([tuple(pt) for pt in arr[:, :2]], fill=1)
|
||||
return np.array(img.resize((cam_w, cam_h), Image.NEAREST), dtype=bool)
|
||||
|
||||
|
||||
def build_disc_lookup(manifest_path: Path) -> dict:
|
||||
mf = pd.read_csv(manifest_path)
|
||||
lookup: dict = {}
|
||||
for _, row in mf.iterrows():
|
||||
sid = str(row["sample_id"])
|
||||
if not sid.startswith("papila_RET"):
|
||||
continue
|
||||
suffix = sid[len("papila_RET"):]
|
||||
eye = suffix[-2:]
|
||||
pid = int(suffix[:-2])
|
||||
disc_path = Path(str(row["annotation_disc"]))
|
||||
img_path = Path(str(row["image_path"]))
|
||||
if not disc_path.exists():
|
||||
continue
|
||||
try:
|
||||
with Image.open(img_path) as im:
|
||||
orig_size = im.size
|
||||
except Exception:
|
||||
continue
|
||||
lookup[(pid, eye)] = (disc_path, orig_size)
|
||||
return lookup
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Disc-centred patch extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def disc_centered_patch(
|
||||
cam: np.ndarray,
|
||||
disc_mask: np.ndarray,
|
||||
span: int = DISC_SPAN,
|
||||
out: int = OUTPUT_SIZE,
|
||||
) -> tuple[np.ndarray | None, float | None]:
|
||||
"""
|
||||
Return (patch, disc_r_out):
|
||||
patch — (out, out) float32 in [0, 1]
|
||||
disc_r_out — disc radius in patch-pixel units (for drawing reference circle)
|
||||
"""
|
||||
if disc_mask is None or disc_mask.sum() == 0:
|
||||
return None, None
|
||||
|
||||
ys, xs = np.where(disc_mask)
|
||||
cy, cx = ys.mean(), xs.mean()
|
||||
disc_r = float(np.sqrt(disc_mask.sum() / np.pi))
|
||||
half = max(1, int(round(span * disc_r / 2)))
|
||||
|
||||
h, w = cam.shape
|
||||
y0, y1 = int(round(cy)) - half, int(round(cy)) + half
|
||||
x0, x1 = int(round(cx)) - half, int(round(cx)) + half
|
||||
|
||||
pt = max(0, -y0); pb = max(0, y1 - h)
|
||||
pl = max(0, -x0); pr = max(0, x1 - w)
|
||||
cam_pad = np.pad(cam, ((pt, pb), (pl, pr)), constant_values=0.0)
|
||||
|
||||
patch = cam_pad[y0 + pt : y1 + pt, x0 + pl : x1 + pl]
|
||||
patch_img = Image.fromarray((np.clip(patch, 0, 1) * 255).astype(np.uint8))
|
||||
patch_out = np.array(patch_img.resize((out, out), Image.BILINEAR)) / 255.0
|
||||
|
||||
disc_r_out = out * disc_r / (2 * half)
|
||||
return patch_out.astype(np.float32), disc_r_out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def load_all_cam_records(mode_dir: Path) -> list[dict]:
|
||||
records = []
|
||||
fold_dirs = sorted(
|
||||
[d for d in mode_dir.iterdir() if d.is_dir() and d.name.startswith("fold")],
|
||||
key=lambda p: int(p.name.replace("fold", "")),
|
||||
)
|
||||
for fd in fold_dirs:
|
||||
gcam_dir = fd / "explainability" / "gradcam"
|
||||
idx_path = gcam_dir / "gradcam_index.csv"
|
||||
if not idx_path.exists():
|
||||
continue
|
||||
idx = pd.read_csv(idx_path)
|
||||
for _, row in idx.iterrows():
|
||||
pid = int(row["patient_id"])
|
||||
for eye in ("OD", "OS"):
|
||||
npy = gcam_dir / f"patient_{pid}_{eye}_cam.npy"
|
||||
if not npy.exists():
|
||||
continue
|
||||
records.append({
|
||||
"patient_id": pid,
|
||||
"eye": eye,
|
||||
"true_name": row["true_name"],
|
||||
"correct": bool(row["correct"]),
|
||||
"cam": np.load(npy),
|
||||
})
|
||||
return records
|
||||
|
||||
|
||||
def build_mean_patches(
|
||||
records: list[dict],
|
||||
disc_lookup: dict,
|
||||
classes: list[str],
|
||||
) -> dict[tuple, tuple]:
|
||||
"""
|
||||
Returns {(cls, split): (mean_patch, mean_disc_r_out, count)}
|
||||
split = 'correct' | 'incorrect'
|
||||
"""
|
||||
buckets: dict[tuple, list] = {}
|
||||
radii: dict[tuple, list] = {}
|
||||
|
||||
for r in records:
|
||||
split = "correct" if r["correct"] else "incorrect"
|
||||
key = (r["true_name"], split)
|
||||
pid, eye = r["patient_id"], r["eye"]
|
||||
if (pid, eye) not in disc_lookup:
|
||||
continue
|
||||
disc_path, orig_size = disc_lookup[(pid, eye)]
|
||||
h, w = r["cam"].shape
|
||||
disc_mask = _load_disc_mask(disc_path, orig_size, h, w)
|
||||
patch, disc_r_out = disc_centered_patch(r["cam"], disc_mask)
|
||||
if patch is None:
|
||||
continue
|
||||
buckets.setdefault(key, []).append(patch)
|
||||
radii.setdefault(key, []).append(disc_r_out)
|
||||
|
||||
return {
|
||||
key: (np.stack(ps).mean(0), float(np.mean(radii[key])), len(ps))
|
||||
for key, ps in buckets.items()
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plotting
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--agg-dir", required=True)
|
||||
ap.add_argument("--manifest", default="manifest.csv")
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
agg_dir = Path(args.agg_dir)
|
||||
mode_dir = agg_dir.parent
|
||||
out_path = Path(args.out) if args.out else agg_dir / "disc_attention_detail.png"
|
||||
|
||||
print("Loading disc lookup…")
|
||||
disc_lookup = build_disc_lookup(Path(args.manifest))
|
||||
print(f" {len(disc_lookup)} entries")
|
||||
|
||||
print("Loading CAM records…")
|
||||
records = load_all_cam_records(mode_dir)
|
||||
print(f" {len(records)} eye records")
|
||||
|
||||
stats = pd.read_csv(agg_dir / "attention_stats.csv")
|
||||
classes = sorted({r["true_name"] for r in records})
|
||||
n_cls = len(classes)
|
||||
print(f"Classes: {classes}")
|
||||
|
||||
print("Building disc-centred mean patches…")
|
||||
mean_patches = build_mean_patches(records, disc_lookup, classes)
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Figure
|
||||
# -----------------------------------------------------------------------
|
||||
corr_colors = {"correct": "steelblue", "incorrect": "tomato"}
|
||||
splits = ["correct", "incorrect"]
|
||||
row_labels = ["Correct", "Incorrect", "Disc fraction\n(strip plot)"]
|
||||
|
||||
fig, axes = plt.subplots(3, n_cls, figsize=(4.2 * n_cls, 13))
|
||||
if n_cls == 1:
|
||||
axes = axes[:, np.newaxis]
|
||||
|
||||
# ---- rows 0 & 1: disc-centred heatmaps ----
|
||||
for ri, split in enumerate(splits):
|
||||
for ci, cls in enumerate(classes):
|
||||
ax = axes[ri, ci]
|
||||
key = (cls, split)
|
||||
if key in mean_patches:
|
||||
mean_patch, disc_r_out, count = mean_patches[key]
|
||||
ax.imshow(mean_patch, cmap="jet", vmin=0, vmax=1, origin="upper",
|
||||
extent=[0, OUTPUT_SIZE, OUTPUT_SIZE, 0])
|
||||
cx = cy = OUTPUT_SIZE / 2
|
||||
ax.add_patch(Circle((cx, cy), disc_r_out,
|
||||
fill=False, edgecolor="white",
|
||||
linewidth=2, linestyle="--"))
|
||||
ax.set_title(f"{cls} | {split}\n(N={count})", fontsize=9)
|
||||
else:
|
||||
ax.text(0.5, 0.5, "no data", ha="center", va="center",
|
||||
transform=ax.transAxes, fontsize=9, color="grey")
|
||||
ax.set_title(f"{cls} | {split}", fontsize=9)
|
||||
ax.axis("off")
|
||||
axes[ri, 0].set_ylabel(row_labels[ri], fontsize=10, labelpad=6)
|
||||
|
||||
# ---- row 2: strip plots ----
|
||||
rng = np.random.default_rng(42)
|
||||
for ci, cls in enumerate(classes):
|
||||
ax = axes[2, ci]
|
||||
sub = stats[stats["true_name"] == cls].dropna(subset=["disc_frac"])
|
||||
|
||||
for xi, split in enumerate(splits):
|
||||
correct_val = (split == "correct")
|
||||
pts = sub[sub["correct"] == correct_val]["disc_frac"].values
|
||||
if len(pts) == 0:
|
||||
continue
|
||||
color = corr_colors[split]
|
||||
jitter = rng.uniform(-0.18, 0.18, size=len(pts))
|
||||
ax.scatter(xi + jitter, pts, color=color, alpha=0.7, s=28, edgecolors="none")
|
||||
ax.hlines(pts.mean(), xi - 0.28, xi + 0.28,
|
||||
colors=color, linewidth=2.5, zorder=5)
|
||||
|
||||
ax.set_xticks([0, 1])
|
||||
ax.set_xticklabels(["Correct", "Incorrect"], fontsize=9)
|
||||
ax.set_xlim(-0.55, 1.55)
|
||||
ax.set_ylim(0, 1)
|
||||
ax.set_title(cls, fontsize=10)
|
||||
ax.grid(axis="y", linestyle="--", alpha=0.3)
|
||||
if ci == 0:
|
||||
ax.set_ylabel("Disc fraction\n(GT disc attention)", fontsize=9)
|
||||
|
||||
fig.suptitle(
|
||||
"Disc-centred GradCAM attention | dashed circle = GT disc boundary",
|
||||
fontsize=12,
|
||||
)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,286 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Visualise aggregated GradCAM heatmaps produced by aggregate_gradcam.py.
|
||||
|
||||
Produces two figures:
|
||||
|
||||
Figure 1 — Mean heatmaps grid
|
||||
Rows: classes (e.g. Normal, Glaucoma)
|
||||
Cols: OD_all | OS_all | OD_correct | OD_incorrect | OS_correct | OS_incorrect
|
||||
|
||||
Figure 2 — Attention stats
|
||||
Panel A: disc_frac distribution per class (violin/box), OD and OS side by side
|
||||
Panel B: entropy distribution per class
|
||||
Panel C: disc_frac correct vs incorrect per class (scatter means + error bars)
|
||||
|
||||
Figure 3 — Disc attention vs correct confidence
|
||||
Scatter of disc_frac vs correct_conf (confidence if correct, 1-confidence if wrong)
|
||||
One panel per class, OD and OS overlaid, Pearson r annotated
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/explainability/plot_gradcam_aggregate.py \
|
||||
--agg-dir analysis_data/pipeline_nocrop/binary/single/gradcam_aggregate
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.patches as mpatches
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def load_agg(agg_dir: Path):
|
||||
npz = np.load(agg_dir / "mean_heatmaps.npz")
|
||||
stats = pd.read_csv(agg_dir / "attention_stats.csv")
|
||||
return npz, stats
|
||||
|
||||
|
||||
def _classes_from_npz(npz) -> list[str]:
|
||||
classes = []
|
||||
for key in npz.files:
|
||||
parts = key.split("_")
|
||||
# key format: {EYE}_{ClassName}_{split}_{stat}
|
||||
# ClassName may be multi-word (e.g. "Glaucoma", "Normal", "Suspect")
|
||||
if parts[-1] == "mean" and parts[-2] == "all" and parts[0] == "OD":
|
||||
classes.append(parts[1])
|
||||
return sorted(set(classes))
|
||||
|
||||
|
||||
def plot_mean_heatmaps(npz, classes: list[str], out_path: Path):
|
||||
eyes = ["OD", "OS"]
|
||||
splits = ["all", "correct", "incorrect"]
|
||||
cols = [(e, s) for e in eyes for s in splits] # 6 columns
|
||||
|
||||
n_rows = len(classes)
|
||||
n_cols = len(cols)
|
||||
fig, axes = plt.subplots(n_rows, n_cols, figsize=(n_cols * 2.8, n_rows * 2.8))
|
||||
if n_rows == 1:
|
||||
axes = axes[np.newaxis, :]
|
||||
|
||||
for r, cls in enumerate(classes):
|
||||
for c, (eye, split) in enumerate(cols):
|
||||
ax = axes[r, c]
|
||||
key = f"{eye}_{cls}_{split}_mean"
|
||||
if key not in npz:
|
||||
ax.axis("off")
|
||||
ax.set_title(f"{eye} {split}\n(no data)", fontsize=7)
|
||||
continue
|
||||
cam = npz[key]
|
||||
count = int(npz.get(f"{eye}_{cls}_{split}_count", np.array(0)))
|
||||
ax.imshow(cam, cmap="jet", vmin=0, vmax=1)
|
||||
ax.axis("off")
|
||||
title = f"{cls} | {eye} {split}\n(N={count})"
|
||||
ax.set_title(title, fontsize=7)
|
||||
|
||||
fig.suptitle("Mean GradCAM heatmaps by class / eye / outcome", fontsize=12)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out_path}")
|
||||
|
||||
|
||||
def plot_attention_stats(stats: pd.DataFrame, classes: list[str], out_path: Path):
|
||||
eyes = ["OD", "OS"]
|
||||
cmap = plt.get_cmap("tab10")
|
||||
class_colors = {cls: cmap(i) for i, cls in enumerate(classes)}
|
||||
|
||||
fig, axes = plt.subplots(1, 3, figsize=(16, 5))
|
||||
|
||||
# ---- Panel A: disc_frac per class × eye ----
|
||||
ax = axes[0]
|
||||
positions = []
|
||||
labels = []
|
||||
data_viol = []
|
||||
tick_pos = []
|
||||
pos = 0
|
||||
for cls in classes:
|
||||
for eye in eyes:
|
||||
sub = stats[(stats["true_name"] == cls) & (stats["eye"] == eye)]["disc_frac"].dropna()
|
||||
data_viol.append(sub.values)
|
||||
positions.append(pos)
|
||||
labels.append(f"{cls[:3]}\n{eye}")
|
||||
tick_pos.append(pos)
|
||||
pos += 1
|
||||
pos += 0.5 # gap between classes
|
||||
|
||||
vp = ax.violinplot(data_viol, positions=positions, showmedians=True, widths=0.7)
|
||||
for i, (pc, cls) in enumerate(zip(vp["bodies"], [c for c in classes for _ in eyes])):
|
||||
pc.set_facecolor(class_colors[cls])
|
||||
pc.set_alpha(0.65)
|
||||
ax.set_xticks(tick_pos)
|
||||
ax.set_xticklabels(labels, fontsize=8)
|
||||
ax.set_ylabel("Disc fraction (attention mass within GT disc mask)")
|
||||
ax.set_title("Disc attention by class")
|
||||
ax.axhline(0.5, color="black", linewidth=0.8, linestyle="--", alpha=0.4)
|
||||
ax.grid(axis="y", linestyle="--", alpha=0.3)
|
||||
|
||||
# ---- Panel B: entropy per class × eye (same layout) ----
|
||||
ax = axes[1]
|
||||
data_ent = []
|
||||
for cls in classes:
|
||||
for eye in eyes:
|
||||
sub = stats[(stats["true_name"] == cls) & (stats["eye"] == eye)]["entropy"].dropna()
|
||||
data_ent.append(sub.values)
|
||||
|
||||
vp2 = ax.violinplot(data_ent, positions=positions, showmedians=True, widths=0.7)
|
||||
for pc, cls in zip(vp2["bodies"], [c for c in classes for _ in eyes]):
|
||||
pc.set_facecolor(class_colors[cls])
|
||||
pc.set_alpha(0.65)
|
||||
ax.set_xticks(tick_pos)
|
||||
ax.set_xticklabels(labels, fontsize=8)
|
||||
ax.set_ylabel("Attention entropy (higher = more diffuse)")
|
||||
ax.set_title("Attention entropy by class")
|
||||
ax.grid(axis="y", linestyle="--", alpha=0.3)
|
||||
|
||||
# ---- Panel C: disc_frac correct vs incorrect, mean ± std ----
|
||||
ax = axes[2]
|
||||
x_ticks = []
|
||||
x_labels = []
|
||||
pos = 0
|
||||
for cls in classes:
|
||||
for eye in eyes:
|
||||
for split, marker, ls in [("correct", "o", "-"), ("incorrect", "X", "--")]:
|
||||
sub = stats[
|
||||
(stats["true_name"] == cls) &
|
||||
(stats["eye"] == eye) &
|
||||
(stats["correct"] == (split == "correct"))
|
||||
]["disc_frac"].dropna()
|
||||
if len(sub) == 0:
|
||||
continue
|
||||
ax.errorbar(
|
||||
pos, sub.mean(), yerr=sub.std(),
|
||||
fmt=marker, color=class_colors[cls], linestyle=ls,
|
||||
capsize=4, markersize=7, alpha=0.85,
|
||||
label=f"{cls[:3]} {eye} {split}" if pos < 4 else "_",
|
||||
)
|
||||
pos += 1
|
||||
x_ticks.append(pos - 1.5)
|
||||
x_labels.append(f"{cls[:3]}\n{eye}")
|
||||
pos += 0.5
|
||||
|
||||
ax.axhline(0.5, color="black", linewidth=0.8, linestyle="--", alpha=0.4)
|
||||
ax.set_ylabel("Disc fraction")
|
||||
ax.set_title("Disc fraction: correct vs incorrect\n(circle=correct, X=incorrect)")
|
||||
ax.grid(axis="y", linestyle="--", alpha=0.3)
|
||||
|
||||
# legend: one patch per class
|
||||
patches = [mpatches.Patch(color=class_colors[c], label=c) for c in classes]
|
||||
patches += [
|
||||
plt.Line2D([0], [0], marker="o", color="grey", label="correct", linestyle="none"),
|
||||
plt.Line2D([0], [0], marker="X", color="grey", label="incorrect", linestyle="none"),
|
||||
]
|
||||
ax.legend(handles=patches, fontsize=7, loc="lower right")
|
||||
|
||||
fig.suptitle("GradCAM attention statistics", fontsize=12)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out_path}")
|
||||
|
||||
|
||||
def plot_disc_attention_correlation(stats: pd.DataFrame, classes: list[str], out_path: Path):
|
||||
"""
|
||||
Scatter disc_frac vs correct_conf per class.
|
||||
|
||||
correct_conf = confidence if correct
|
||||
= 1 - confidence if incorrect
|
||||
|
||||
This asks: does focusing attention on the disc region correlate with
|
||||
the model being more confident about the right answer?
|
||||
"""
|
||||
import scipy.stats as scipy_stats
|
||||
|
||||
stats = stats.copy()
|
||||
stats["correct_conf"] = np.where(
|
||||
stats["correct"],
|
||||
stats["confidence"],
|
||||
1.0 - stats["confidence"],
|
||||
)
|
||||
|
||||
corr_colors = {True: "steelblue", False: "tomato"}
|
||||
corr_labels = {True: "Correct", False: "Incorrect"}
|
||||
is_binary = len(classes) == 2
|
||||
|
||||
n_cls = len(classes)
|
||||
fig, axes = plt.subplots(1, n_cls, figsize=(5 * n_cls, 5), sharey=True)
|
||||
if n_cls == 1:
|
||||
axes = [axes]
|
||||
|
||||
for ax, cls in zip(axes, classes):
|
||||
sub = stats[stats["true_name"] == cls]
|
||||
x_all, y_all = [], []
|
||||
|
||||
for correct_val, color in corr_colors.items():
|
||||
csub = sub[sub["correct"] == correct_val]
|
||||
x = csub["disc_frac"].values
|
||||
y = csub["correct_conf"].values
|
||||
ax.scatter(x, y, marker="o", color=color,
|
||||
alpha=0.75, s=30,
|
||||
label=corr_labels[correct_val],
|
||||
edgecolors="none")
|
||||
x_all.extend(x.tolist())
|
||||
y_all.extend(y.tolist())
|
||||
|
||||
# pooled regression line
|
||||
x_arr = np.array(x_all)
|
||||
y_arr = np.array(y_all)
|
||||
if len(x_arr) >= 3:
|
||||
r, p = scipy_stats.pearsonr(x_arr, y_arr)
|
||||
m, b = np.polyfit(x_arr, y_arr, 1)
|
||||
xs = np.linspace(0, 1, 100)
|
||||
ax.plot(xs, m * xs + b, color="black", linewidth=1.5, linestyle="--", alpha=0.7)
|
||||
p_str = f"p={p:.3f}" if p >= 0.001 else "p<0.001"
|
||||
ax.annotate(f"r={r:+.3f}\n{p_str}", xy=(0.05, 0.93), xycoords="axes fraction",
|
||||
fontsize=9, va="top",
|
||||
bbox=dict(boxstyle="round,pad=0.3", facecolor="white", alpha=0.7))
|
||||
|
||||
if is_binary:
|
||||
ax.axhline(0.5, color="red", linewidth=1.0, linestyle=":",
|
||||
alpha=0.7, label="Decision boundary (0.50)")
|
||||
ax.set_xlim(0, 1)
|
||||
ax.set_xlabel("Disc fraction\n(attention mass within GT disc mask)", fontsize=9)
|
||||
ax.set_title(cls, fontsize=11)
|
||||
ax.set_ylim(-0.02, 1.05)
|
||||
ax.grid(linestyle="--", alpha=0.3)
|
||||
ax.legend(fontsize=8, loc="lower right")
|
||||
|
||||
axes[0].set_ylabel("Correct-class confidence\n(conf if correct, 1−conf if wrong)", fontsize=9)
|
||||
fig.suptitle("Disc attention vs correct-class confidence", fontsize=12)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out_path}")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--agg-dir", required=True,
|
||||
help="Directory produced by aggregate_gradcam.py")
|
||||
ap.add_argument("--out-heatmaps", default=None)
|
||||
ap.add_argument("--out-stats", default=None)
|
||||
ap.add_argument("--out-corr", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
agg_dir = Path(args.agg_dir)
|
||||
out_hm = Path(args.out_heatmaps) if args.out_heatmaps else agg_dir / "mean_heatmaps_plot.png"
|
||||
out_st = Path(args.out_stats) if args.out_stats else agg_dir / "attention_stats_plot.png"
|
||||
out_corr = Path(args.out_corr) if args.out_corr else agg_dir / "disc_attention_correlation.png"
|
||||
|
||||
npz, stats = load_agg(agg_dir)
|
||||
classes = _classes_from_npz(npz)
|
||||
print(f"Classes found: {classes}")
|
||||
print(f"Total eye records in stats: {len(stats)}")
|
||||
|
||||
plot_mean_heatmaps(npz, classes, out_hm)
|
||||
plot_attention_stats(stats, classes, out_st)
|
||||
plot_disc_attention_correlation(stats, classes, out_corr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user