Add analysis scripts and experiment configurations for bridge attention and sensitivity studies
- Introduced `bridge_attention_ceiling_check.py` for variance decomposition analysis on bridge attention configurations. - Added `bridge_attention_readout.py` to perform per-tower gate and contribution readouts, including AUC sanity checks. - Created multiple JSON configuration files for backbone replication experiments, including anonymous CV variants and basic backbones. - Implemented sensitivity experiments to evaluate the impact of axial length inclusion and EfficientNetV2-M performance at higher resolutions. - Added a memory probe script to assess GPU memory usage during training with EfficientNetV2-M.
This commit is contained in:
rpotter6298
@@ -1,50 +1,56 @@
|
||||
"""F2 — Backbone selection panel.
|
||||
"""F2 - Backbone selection panel.
|
||||
|
||||
Box plot in the style of v3/figures/phase2_analysis.png (black-bordered boxes,
|
||||
red median lines, baseline median reference). Three left-to-right sections:
|
||||
|
||||
Block 1 (blue) — Basic backbones (img-only, single-eye, ImageNet pretraining):
|
||||
Block 1 (blue) -- Basic backbones (img-only, single-eye, ImageNet pretraining):
|
||||
VGG16, MobileNetV2, DenseNet121, InceptionV3, ResNet50
|
||||
Sourced from v3 phase 1 / phase 2 fold AUCs. Will be refined with v4
|
||||
10x5 runs later; means should not move much.
|
||||
Sourced from v4 experiments/backbone_replication/basic_* (10x5 = 50 fold-rep
|
||||
AUCs each, img-only single-eye, patient-grouped CV).
|
||||
|
||||
Block 2 (blue) — ResNet50 preprocessing/CV variations:
|
||||
leaky CV, GT crop, U-Net crop (all 2.5x scale; 1.1x dropped from labels)
|
||||
Sourced from v3 phase 2 'classic_test_auc' (single-mode image-only).
|
||||
Block 2 (blue) -- ResNet50 preprocessing/CV variations:
|
||||
Anonymous CV, GT crop, U-Net crop (disc crops use margin 2.5x)
|
||||
"Anonymous CV" = patient-identity-agnostic cross-validation: fold
|
||||
assignment ignores PAPILA's patient IDs, allowing the same patient's
|
||||
OD/OS pair to be split across train and test. Reflects the standard
|
||||
protocol in benchmark reports that do not have patient-level labels
|
||||
(or do not respect them).
|
||||
Sourced from v4 experiments/backbone_replication/{anonymous_cv,gtcrop,
|
||||
unetcrop}_refugelike.
|
||||
|
||||
Block 3 (orange) — Baseline reference:
|
||||
"Baseline (fine-tuned ResNet50)" — what we previously called refugelike.
|
||||
Sourced from v3 phase 2 imageonly_refugelike_proper.
|
||||
Block 3 (orange) -- Baseline reference:
|
||||
"Baseline (fine-tuned ResNet50)" -- REFUGE-pretrained R50 image-only,
|
||||
sourced from v4 experiments/refuge_v2m_baseline/img_solo_single_refugelike.
|
||||
|
||||
Each non-baseline box is labelled with a Wilcoxon two-sided p-value comparing
|
||||
its fold AUCs to the baseline.
|
||||
its fold-rep AUCs to the baseline fold-rep AUCs.
|
||||
|
||||
Re-run anytime:
|
||||
python -m v4.figures.F2_papila_replication_and_single_mode
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from scipy.stats import wilcoxon
|
||||
|
||||
from v4.figures.util.loaders import REPO_ROOT
|
||||
from v4.figures.util.loaders import RESULTS_ROOT
|
||||
|
||||
OUT = Path(__file__).parent / "output" / "F2_backbones.png"
|
||||
|
||||
# ── Colors / styling (mirrors v3 phase2_analysis) ────────────────────────────
|
||||
C_VAR = "#4c72b0" # blue — non-baseline boxes (basic backbones + variants)
|
||||
C_BASE = "#dd8452" # orange — baseline reference box
|
||||
C_MEDIAN = "#c44e52" # red — median line inside boxes
|
||||
# Colors / styling (mirrors v3 phase2_analysis)
|
||||
C_VAR = "#4c72b0" # blue - non-baseline boxes
|
||||
C_BASE = "#dd8452" # orange - baseline reference box
|
||||
C_MEDIAN = "#c44e52" # red - median line inside boxes
|
||||
ALPHA = 0.82
|
||||
|
||||
V3_PHASE1_DIR = REPO_ROOT / "v3" / "results" / "phase1"
|
||||
V3_PHASE2_DIR = REPO_ROOT / "v3" / "results" / "phase2"
|
||||
# Stage key for img-only single-eye fusion
|
||||
STAGE_KEY = "img_fuse_test_auc"
|
||||
|
||||
|
||||
def _wilcoxon_p(a: np.ndarray, b: np.ndarray) -> float:
|
||||
@@ -57,67 +63,64 @@ def _wilcoxon_p(a: np.ndarray, b: np.ndarray) -> float:
|
||||
return float("nan")
|
||||
|
||||
|
||||
def _load_phase1_fold_aucs(subdir: str) -> np.ndarray:
|
||||
fp = V3_PHASE1_DIR / subdir / "fold_metrics.csv"
|
||||
if not fp.exists():
|
||||
return np.array([])
|
||||
df = pd.read_csv(fp)
|
||||
return df["auc"].dropna().astype(float).values
|
||||
|
||||
|
||||
def _load_phase2_classic_aucs(run_name: str) -> np.ndarray:
|
||||
"""Collect classic_test_auc across all rep×fold for a phase 2 run folder."""
|
||||
root = V3_PHASE2_DIR / run_name
|
||||
def _load_fold_aucs(rel: str) -> np.ndarray:
|
||||
"""Collect STAGE_KEY across all rep x fold for a v4 results subdirectory."""
|
||||
root = RESULTS_ROOT / rel
|
||||
if not root.exists():
|
||||
return np.array([])
|
||||
out: list[float] = []
|
||||
for rep in sorted(root.glob("rep*")):
|
||||
fp = rep / "binary" / "single" / "fold_results.csv"
|
||||
if not fp.exists(): continue
|
||||
df = pd.read_csv(fp)
|
||||
if "classic_test_auc" not in df.columns: continue
|
||||
out.extend(df["classic_test_auc"].dropna().astype(float).tolist())
|
||||
for s in sorted(root.glob("rep*/binary/summary.json")):
|
||||
d = json.loads(s.read_text())
|
||||
for fr in d.get("fold_results", []):
|
||||
v = fr.get(STAGE_KEY)
|
||||
if v is None or not np.isfinite(v):
|
||||
continue
|
||||
out.append(float(v))
|
||||
return np.array(out)
|
||||
|
||||
|
||||
# ── Per-section data definitions ─────────────────────────────────────────────
|
||||
# Each entry: (label, loader_fn, *args)
|
||||
# Per-section data definitions (label, results-subdir under RESULTS_ROOT)
|
||||
|
||||
BASIC_BACKBONES = [
|
||||
("VGG16", _load_phase1_fold_aucs, "cnn_vgg16"),
|
||||
("MobileNetV2", _load_phase1_fold_aucs, "cnn_mobilenet_v2"),
|
||||
("DenseNet121", _load_phase1_fold_aucs, "cnn_densenet121"),
|
||||
("InceptionV3", _load_phase1_fold_aucs, "cnn_inception_v3"),
|
||||
# Use phase 2 ResNet50 (50 fold AUCs) for tighter statistics on the
|
||||
# backbone that we sweep variations of in block 2.
|
||||
("ResNet50", _load_phase2_classic_aucs, "imageonly_resnet50_proper"),
|
||||
("VGG16", "backbone_replication/basic_vgg16"),
|
||||
("MobileNetV2", "backbone_replication/basic_mobilenet_v2"),
|
||||
("DenseNet121", "backbone_replication/basic_densenet121"),
|
||||
("InceptionV3", "backbone_replication/basic_inception_v3"),
|
||||
("ResNet50", "backbone_replication/basic_resnet50"),
|
||||
]
|
||||
|
||||
RESNET_VARIATIONS = [
|
||||
("leaky CV", _load_phase2_classic_aucs, "imageonly_resnet50_leaky"),
|
||||
("GT crop", _load_phase2_classic_aucs, "imageonly_resnet50_gtcrop_2.5"),
|
||||
("U-Net crop", _load_phase2_classic_aucs, "imageonly_resnet50_unetcrop_2.5"),
|
||||
("Anonymous CV", "backbone_replication/anonymous_cv_refugelike"),
|
||||
("GT crop", "backbone_replication/gtcrop_refugelike"),
|
||||
("U-Net crop", "backbone_replication/unetcrop_refugelike"),
|
||||
]
|
||||
|
||||
BASELINE_LABEL = "baseline\n(fine-tuned ResNet50)"
|
||||
BASELINE_DATA = (_load_phase2_classic_aucs, "imageonly_refugelike_proper")
|
||||
BASELINE_REL = "refuge_v2m_baseline/img_solo_single_refugelike"
|
||||
|
||||
|
||||
def render() -> None:
|
||||
# Load everything
|
||||
block1 = [(lbl, fn(arg)) for lbl, fn, arg in BASIC_BACKBONES]
|
||||
block2 = [(lbl, fn(arg)) for lbl, fn, arg in RESNET_VARIATIONS]
|
||||
base_fn, base_arg = BASELINE_DATA
|
||||
base_aucs = base_fn(base_arg)
|
||||
block1 = [(lbl, _load_fold_aucs(rel)) for lbl, rel in BASIC_BACKBONES]
|
||||
block2 = [(lbl, _load_fold_aucs(rel)) for lbl, rel in RESNET_VARIATIONS]
|
||||
base_aucs = _load_fold_aucs(BASELINE_REL)
|
||||
|
||||
print("Block 1 — Basic backbones:")
|
||||
print("Block 1 - Basic backbones (ImageNet pretraining):")
|
||||
for lbl, a in block1:
|
||||
print(f" {lbl:<14s} n={len(a):>3d} mean={a.mean():.3f}±{a.std():.3f}" if len(a) else f" {lbl:<14s} no data")
|
||||
print("Block 2 — ResNet50 variations:")
|
||||
if len(a):
|
||||
print(f" {lbl:<14s} n={len(a):>3d} mean={a.mean():.4f} +/- {a.std():.4f}")
|
||||
else:
|
||||
print(f" {lbl:<14s} no data")
|
||||
print("Block 2 - ResNet50 (REFUGE) variations:")
|
||||
for lbl, a in block2:
|
||||
print(f" {lbl:<14s} n={len(a):>3d} mean={a.mean():.3f}±{a.std():.3f}" if len(a) else f" {lbl:<14s} no data")
|
||||
print(f"Block 3 — Baseline: n={len(base_aucs)} "
|
||||
f"mean={base_aucs.mean():.3f}±{base_aucs.std():.3f}" if len(base_aucs) else "Block 3 — no baseline data")
|
||||
if len(a):
|
||||
print(f" {lbl:<14s} n={len(a):>3d} mean={a.mean():.4f} +/- {a.std():.4f}")
|
||||
else:
|
||||
print(f" {lbl:<14s} no data")
|
||||
if len(base_aucs):
|
||||
print(f"Block 3 - Baseline: n={len(base_aucs)} "
|
||||
f"mean={base_aucs.mean():.4f} +/- {base_aucs.std():.4f}")
|
||||
else:
|
||||
print("Block 3 - no baseline data")
|
||||
|
||||
# Lay out positions
|
||||
gap = 0.7
|
||||
@@ -135,7 +138,6 @@ def render() -> None:
|
||||
section3_left = p
|
||||
pos.append(p)
|
||||
section3_right = p
|
||||
total_w = p + 0.6
|
||||
|
||||
fig, ax = plt.subplots(figsize=(13, 5.8))
|
||||
fig.suptitle("Backbone Selection", fontsize=13, fontweight="bold")
|
||||
@@ -156,10 +158,10 @@ def render() -> None:
|
||||
all_labels.append(lbl); all_aucs.append(a); all_colors.append(C_VAR)
|
||||
all_labels.append(BASELINE_LABEL); all_aucs.append(base_aucs); all_colors.append(C_BASE)
|
||||
|
||||
# Draw boxes
|
||||
for x, aucs, color in zip(pos, all_aucs, all_colors):
|
||||
if not len(aucs): continue
|
||||
bp = ax.boxplot(
|
||||
if not len(aucs):
|
||||
continue
|
||||
ax.boxplot(
|
||||
aucs, positions=[x], widths=box_w, patch_artist=True, manage_ticks=False,
|
||||
boxprops=dict(facecolor=color, alpha=ALPHA, **boxprops_kw),
|
||||
medianprops=medianprops,
|
||||
@@ -168,31 +170,27 @@ def render() -> None:
|
||||
flierprops=flierprops,
|
||||
)
|
||||
|
||||
# Baseline median reference line spanning the variant blocks
|
||||
if len(base_aucs):
|
||||
ax.axhline(np.median(base_aucs),
|
||||
color=C_BASE, linewidth=1.2, linestyle="--", alpha=0.55,
|
||||
label="Baseline median")
|
||||
|
||||
# Dividers between sections (vertical light lines)
|
||||
div1 = (section1_right + section2_left) / 2
|
||||
div2 = (section2_right + section3_left) / 2
|
||||
for d in (div1, div2):
|
||||
ax.axvline(d, color="#aaa", linewidth=0.7, alpha=0.65, linestyle="-")
|
||||
|
||||
# Section labels just above each block
|
||||
y_band = 1.02
|
||||
section_centers = [
|
||||
((pos[0] + section1_right) / 2, "Basic backbones (img-only, single)"),
|
||||
((section2_left + section2_right) / 2, "ResNet50 variations"),
|
||||
((section3_left + section3_right) / 2, "Baseline"),
|
||||
((pos[0] + section1_right) / 2, "Basic backbones (img-only, single)"),
|
||||
((section2_left + section2_right) / 2, "ResNet50 variations"),
|
||||
((section3_left + section3_right) / 2, "Baseline"),
|
||||
]
|
||||
for cx, txt in section_centers:
|
||||
ax.text(cx, y_band, txt, ha="center", va="bottom",
|
||||
fontsize=10, color="#333", fontweight="bold",
|
||||
transform=ax.get_xaxis_transform())
|
||||
|
||||
# X-tick labels (with p-values vs baseline beneath each variant box)
|
||||
tick_labels = []
|
||||
for lbl, aucs, color in zip(all_labels, all_aucs, all_colors):
|
||||
if color == C_BASE or not len(aucs) or not len(base_aucs):
|
||||
|
||||
Reference in New Issue
Block a user