Add analysis scripts and experiment configurations for bridge attention and sensitivity studies
- Introduced `bridge_attention_ceiling_check.py` for variance decomposition analysis on bridge attention configurations. - Added `bridge_attention_readout.py` to perform per-tower gate and contribution readouts, including AUC sanity checks. - Created multiple JSON configuration files for backbone replication experiments, including anonymous CV variants and basic backbones. - Implemented sensitivity experiments to evaluate the impact of axial length inclusion and EfficientNetV2-M performance at higher resolutions. - Added a memory probe script to assess GPU memory usage during training with EfficientNetV2-M.
This commit is contained in:
rpotter6298
@@ -0,0 +1,117 @@
|
||||
"""Variance decomposition: is the high-backbone end of the bridge_attention
|
||||
sweep hitting a dataset ceiling?
|
||||
|
||||
For each (rep, fold) cell, we have 12 hb_test_auc measurements — one per
|
||||
(bridge × backbone) condition. We decompose the variance two ways and compare
|
||||
between the LOW-backbone and HIGH-backbone halves of the gradient:
|
||||
|
||||
across-arch variance @ fixed (rep,fold)
|
||||
= var across the conditions in this subset, for the same fold split
|
||||
(small → architectures are interchangeable at this capacity)
|
||||
across-fold variance @ fixed architecture
|
||||
= var across the 50 fold-reps, for one condition
|
||||
(small → the fold split doesn't matter much)
|
||||
|
||||
If at the high-backbone end, across-fold dwarfs across-arch, the dataset's
|
||||
fold-assignment noise dominates the architectural choice — i.e. all the
|
||||
strong configurations are hitting the same ceiling.
|
||||
|
||||
Reads summary.json files directly, no inference needed.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
SWEEP_ROOT = Path("v4/results/experiments/bridge_attention")
|
||||
|
||||
LOW_BACKBONES = ["mobilenet_v2", "resnet50", "efficientnet_b0"]
|
||||
HIGH_BACKBONES = ["efficientnet_v2_m", "refugelike", "refuge_efficientnet_v2_m"]
|
||||
BRIDGES = ["gated", "ortho"]
|
||||
|
||||
|
||||
def collect_long() -> pd.DataFrame:
|
||||
rows = []
|
||||
for bridge in BRIDGES:
|
||||
for bb in LOW_BACKBONES + HIGH_BACKBONES:
|
||||
run_dir = SWEEP_ROOT / f"{bridge}_{bb}"
|
||||
if not run_dir.exists():
|
||||
continue
|
||||
for s in sorted(run_dir.glob("rep*/binary/summary.json")):
|
||||
rep = int(s.parents[1].name.replace("rep", ""))
|
||||
d = json.loads(s.read_text())
|
||||
for fr in d.get("fold_results", []):
|
||||
v = fr.get("hb_test_auc")
|
||||
if v is None or not np.isfinite(v):
|
||||
continue
|
||||
rows.append({
|
||||
"bridge": bridge,
|
||||
"backbone": bb,
|
||||
"rep": rep,
|
||||
"fold": fr["fold"],
|
||||
"auc": float(v),
|
||||
})
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def decompose(df: pd.DataFrame, label: str) -> None:
|
||||
# condition = bridge × backbone tuple
|
||||
df = df.copy()
|
||||
df["condition"] = df["bridge"] + "/" + df["backbone"]
|
||||
n_cond = df["condition"].nunique()
|
||||
n_cells = df.groupby(["rep", "fold"]).ngroups
|
||||
|
||||
# Across-architecture variance @ fixed (rep, fold)
|
||||
grouped_cell = df.groupby(["rep", "fold"])["auc"]
|
||||
cell_var = grouped_cell.var(ddof=1) # one var per (rep,fold) cell
|
||||
cell_std_mean = float(np.sqrt(cell_var.mean())) if not cell_var.empty else float("nan")
|
||||
|
||||
# Across-fold-rep variance @ fixed architecture
|
||||
grouped_arch = df.groupby("condition")["auc"]
|
||||
arch_var = grouped_arch.var(ddof=1)
|
||||
arch_std_mean = float(np.sqrt(arch_var.mean())) if not arch_var.empty else float("nan")
|
||||
|
||||
ratio = arch_std_mean / cell_std_mean if cell_std_mean > 0 else float("inf")
|
||||
mean_auc = float(df["auc"].mean())
|
||||
print(f"── {label} ── (n_conditions={n_cond}, n_cells={n_cells}, n_obs={len(df)})")
|
||||
print(f" mean AUC across all (cond, rep, fold) ........ {mean_auc:.4f}")
|
||||
print(f" across-arch SD @ fixed (rep,fold) ............ {cell_std_mean:.4f} "
|
||||
f"← architectural spread within the same fold")
|
||||
print(f" across-foldrep SD @ fixed architecture ....... {arch_std_mean:.4f} "
|
||||
f"← fold-assignment noise within a single architecture")
|
||||
print(f" ratio arch-SD / fold-SD ..................... {ratio:>6.2f}x "
|
||||
f"({'fold noise dominates' if ratio > 2 else 'arch + fold comparable'})")
|
||||
print()
|
||||
|
||||
|
||||
def main():
|
||||
df = collect_long()
|
||||
if df.empty:
|
||||
print("No data — has the full readout finished yet?")
|
||||
return
|
||||
|
||||
n_cond = df["bridge"].nunique() * df["backbone"].nunique()
|
||||
print(f"Collected {len(df)} (cond, rep, fold) AUC observations from "
|
||||
f"{n_cond} (bridge × backbone) conditions\n")
|
||||
|
||||
low = df[df["backbone"].isin(LOW_BACKBONES)]
|
||||
high = df[df["backbone"].isin(HIGH_BACKBONES)]
|
||||
decompose(low, "LOW backbones (mobilenet_v2, resnet50, efficientnet_b0)")
|
||||
decompose(high, "HIGH backbones (efficientnet_v2_m, refugelike, refuge_v2m)")
|
||||
|
||||
# Headline interpretation
|
||||
print("──────────────────────────────────────────────────────")
|
||||
print("Interpretation:")
|
||||
print(" - If both subsets show high arch-SD: architectures genuinely differ.")
|
||||
print(" - If LOW shows high arch-SD but HIGH shows low arch-SD: ceiling effect")
|
||||
print(" at the high-backbone end — all strong configurations hit the same wall.")
|
||||
print(" - Within each subset, ratio = fold-SD / arch-SD: when fold noise")
|
||||
print(" dominates by >2x, the cell-to-cell variation between architectures")
|
||||
print(" is smaller than the noise floor introduced by patient assignment.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,300 @@
|
||||
"""Per-tower gate / contribution readout + per-head AUC sanity check.
|
||||
|
||||
For each gated_<backbone> run (and ortho_<backbone>), loads each fold's
|
||||
checkpoints and runs inference on the test set, capturing in a single pass:
|
||||
• per-sample per-stream bridge attention (sigmoid gates for
|
||||
GatedAdditiveBridge; ||W_i·z_i|| pre-LN projection norms for OrthoBridge)
|
||||
• per-fold AUCs of the eye-level aux heads (img_aux, cd_aux, nt_head) —
|
||||
sanity check that the image tower really is getting stronger as the
|
||||
backbone improves, and that the clinical tower stays consistent
|
||||
|
||||
Outputs:
|
||||
v4/results/experiments/bridge_attention/_summary/gated_gates.csv
|
||||
v4/results/experiments/bridge_attention/_summary/ortho_stream_norms.csv
|
||||
Both CSVs include img_aux_auc, cd_aux_auc, nt_head_auc columns.
|
||||
|
||||
Run:
|
||||
python -m v4.scripts.analysis.bridge_attention_readout
|
||||
python -m v4.scripts.analysis.bridge_attention_readout --bridges gated
|
||||
python -m v4.scripts.analysis.bridge_attention_readout --reps 3 # subsample for speed
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
from tqdm import tqdm
|
||||
|
||||
from v4.classes.split_manager import SplitManager
|
||||
from v4.classes.v4_hypertower import _make_loader, build_towers, load_data
|
||||
import v4.classes.v4_hypertower as orch
|
||||
|
||||
|
||||
SWEEP_ROOT = Path("v4/results/experiments/bridge_attention")
|
||||
OUT_DIR = SWEEP_ROOT / "_summary"
|
||||
|
||||
BACKBONES = [
|
||||
"mobilenet_v2",
|
||||
"resnet50",
|
||||
"efficientnet_b0",
|
||||
"efficientnet_v2_m",
|
||||
"refugelike",
|
||||
"refuge_efficientnet_v2_m",
|
||||
]
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Shared fold-module loader (trimmed version of F8's helper)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _load_fold_bridge(ckpt_dir: Path, cfg: dict, data, device):
|
||||
"""Load towers + nt bridge + eye-level aux heads from a single fold ckpt dir."""
|
||||
from v4.classes.heads.classifier import ClassificationHead
|
||||
|
||||
towers = build_towers(cfg["towers"], data)
|
||||
for name, tower in towers.items():
|
||||
tower.load_state_dict(
|
||||
torch.load(ckpt_dir / f"tower_{name}.pt", map_location="cpu")
|
||||
)
|
||||
tower.to(device).eval()
|
||||
|
||||
stage_by_name = {s["name"]: s for s in cfg["stages"]}
|
||||
nt_cfg = stage_by_name["nt"]
|
||||
nt_mod = importlib.import_module(nt_cfg["module"])
|
||||
nt = getattr(nt_mod, nt_cfg["class"])(
|
||||
[towers[n].out_dim for n in nt_cfg["inputs"]],
|
||||
**nt_cfg.get("args", {}),
|
||||
).to(device)
|
||||
nt.load_state_dict(torch.load(ckpt_dir / "stage_nt.pt", map_location="cpu"))
|
||||
nt.eval()
|
||||
|
||||
nc = cfg["num_classes"]
|
||||
img_aux = ClassificationHead(towers["img"].out_dim, nc).to(device)
|
||||
cd_aux = ClassificationHead(towers["cd"].out_dim, nc).to(device)
|
||||
nt_head = ClassificationHead(nt.out_dim, nc).to(device)
|
||||
img_aux.load_state_dict(torch.load(ckpt_dir / "stage_img_aux.pt", map_location="cpu"))
|
||||
cd_aux.load_state_dict(torch.load(ckpt_dir / "stage_cd_aux.pt", map_location="cpu"))
|
||||
nt_head.load_state_dict(torch.load(ckpt_dir / "stage_nt_head.pt", map_location="cpu"))
|
||||
img_aux.eval(); cd_aux.eval(); nt_head.eval()
|
||||
aux = {"img_aux": img_aux, "cd_aux": cd_aux, "nt_head": nt_head}
|
||||
return towers, nt, nt_cfg, aux
|
||||
|
||||
|
||||
def _splits_from_cfg(cfg: dict, data):
|
||||
label_filter = cfg.get("label_filter", None)
|
||||
df_mode = data.df.copy()
|
||||
if label_filter is not None:
|
||||
df_mode = df_mode[df_mode[data.label_col].isin(label_filter)].reset_index(drop=True)
|
||||
identity_cols = getattr(data, "identity_cols", [])
|
||||
identity_level = cfg.get("split_identity_level", 1)
|
||||
group_col = (
|
||||
identity_cols[identity_level - 1]
|
||||
if identity_level and identity_cols
|
||||
else None
|
||||
)
|
||||
return SplitManager(group_col=group_col).build_plans(
|
||||
df_mode,
|
||||
label_col=data.label_col,
|
||||
n_splits=cfg.get("folds", 5),
|
||||
seed=cfg.get("fold_seed", 100),
|
||||
), label_filter
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Unified inference: capture bridge attention + per-head logits in one pass
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _install_attention_hooks(nt, bridge_kind: str):
|
||||
"""Register the right hook(s) for the bridge kind. Returns (handles, finalize)
|
||||
where finalize() reads back the captured per-sample (N, n_streams) array."""
|
||||
if bridge_kind == "gated":
|
||||
captured: list[torch.Tensor] = []
|
||||
def hook(_module, _input, output):
|
||||
captured.append(output.detach().cpu())
|
||||
handles = [nt.gate.register_forward_hook(hook)]
|
||||
def finalize():
|
||||
return (
|
||||
torch.cat(captured, dim=0).numpy() if captured else np.empty((0, 0))
|
||||
)
|
||||
return handles, finalize
|
||||
|
||||
if bridge_kind == "ortho":
|
||||
# Pre-LN projection norm ||W_i z_i||_2 per stream. Post-LN norm is
|
||||
# √fusion_dim by construction, so we hook the linear projection itself.
|
||||
per_stream: list[list[torch.Tensor]] = [[] for _ in range(len(nt.inner.W))]
|
||||
handles = []
|
||||
for i, w in enumerate(nt.inner.W):
|
||||
def make_hook(idx):
|
||||
def hook(_module, _input, output):
|
||||
per_stream[idx].append(output.detach().norm(dim=-1).cpu())
|
||||
return hook
|
||||
handles.append(w.register_forward_hook(make_hook(i)))
|
||||
def finalize():
|
||||
cols = [
|
||||
torch.cat(c, dim=0).numpy() if c else np.array([])
|
||||
for c in per_stream
|
||||
]
|
||||
if not all(s.size for s in cols):
|
||||
return np.empty((0, 0))
|
||||
return np.stack(cols, axis=1)
|
||||
return handles, finalize
|
||||
|
||||
raise ValueError(f"unknown bridge_kind {bridge_kind!r}")
|
||||
|
||||
|
||||
def extract_one_fold(
|
||||
bridge_kind: str,
|
||||
towers, nt, aux: dict,
|
||||
data, split_obj, label_filter, device,
|
||||
):
|
||||
"""Single inference pass that returns:
|
||||
attention : ndarray (N, n_streams) — bridge-specific attention signal
|
||||
head_aucs : dict[str, float] — AUC of softmax[:,1] for each aux head
|
||||
n_samples : int
|
||||
"""
|
||||
from sklearn.metrics import roc_auc_score
|
||||
import torch.nn.functional as F
|
||||
|
||||
if split_obj.test is None:
|
||||
return np.empty((0, 0)), {}, 0
|
||||
|
||||
shell = data.build_shells(
|
||||
split_obj.test, level="patient", label_filter=label_filter
|
||||
)
|
||||
loader = _make_loader(shell, towers, batch_size=8, shuffle=False)
|
||||
|
||||
handles, finalize = _install_attention_hooks(nt, bridge_kind)
|
||||
img_logits, cd_logits, nt_logits, ys = [], [], [], []
|
||||
|
||||
try:
|
||||
with torch.no_grad():
|
||||
for batch in loader:
|
||||
y = batch["label"].detach().cpu().numpy()
|
||||
for side in ("a", "b"):
|
||||
z_img = towers["img"](batch["img"][side].to(device))
|
||||
z_cd = towers["cd"](batch["cd"][side].to(device))
|
||||
img_logits.append(aux["img_aux"](z_img).cpu())
|
||||
cd_logits.append(aux["cd_aux"](z_cd).cpu())
|
||||
z_fused = nt([z_img, z_cd]) # attention hooks capture here
|
||||
nt_logits.append(aux["nt_head"](z_fused).cpu())
|
||||
ys.append(y)
|
||||
attention = finalize()
|
||||
finally:
|
||||
for h in handles:
|
||||
h.remove()
|
||||
|
||||
if not ys:
|
||||
return attention, {}, 0
|
||||
|
||||
y_all = np.concatenate(ys)
|
||||
head_aucs: dict[str, float] = {}
|
||||
for name, buf in [
|
||||
("img_aux", img_logits),
|
||||
("cd_aux", cd_logits),
|
||||
("nt_head", nt_logits),
|
||||
]:
|
||||
logits = torch.cat(buf, dim=0).numpy()
|
||||
probs = (np.exp(logits - logits.max(axis=1, keepdims=True))
|
||||
/ np.exp(logits - logits.max(axis=1, keepdims=True))
|
||||
.sum(axis=1, keepdims=True))
|
||||
try:
|
||||
head_aucs[name] = float(roc_auc_score(y_all, probs[:, 1]))
|
||||
except Exception:
|
||||
head_aucs[name] = float("nan")
|
||||
|
||||
return attention, head_aucs, int(attention.shape[0])
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Driver
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def run_for_bridge(bridge_kind: str, max_reps: int | None, device) -> pd.DataFrame:
|
||||
rows: list[dict] = []
|
||||
for backbone in BACKBONES:
|
||||
run_dir = SWEEP_ROOT / f"{bridge_kind}_{backbone}"
|
||||
if not run_dir.exists():
|
||||
print(f" [{bridge_kind}/{backbone}] (missing)")
|
||||
continue
|
||||
rep_dirs = sorted(run_dir.glob("rep*"))
|
||||
if max_reps is not None:
|
||||
rep_dirs = rep_dirs[:max_reps]
|
||||
for rep_dir in tqdm(rep_dirs, desc=f"{bridge_kind}/{backbone}", unit="rep"):
|
||||
summary_path = rep_dir / "binary" / "summary.json"
|
||||
if not summary_path.exists():
|
||||
continue
|
||||
cfg = json.loads(summary_path.read_text())["config"]
|
||||
orch.cfg_ref = cfg
|
||||
data = load_data(cfg)
|
||||
splits, label_filter = _splits_from_cfg(cfg, data)
|
||||
|
||||
try:
|
||||
rep_idx = int(rep_dir.name.replace("rep", ""))
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
for fold_idx in range(cfg.get("folds", 5)):
|
||||
ckpt_dir = rep_dir / "binary" / "checkpoints" / f"fold{fold_idx}"
|
||||
if not ckpt_dir.exists():
|
||||
continue
|
||||
towers, nt, _, aux = _load_fold_bridge(ckpt_dir, cfg, data, device)
|
||||
arr, head_aucs, _ = extract_one_fold(
|
||||
bridge_kind, towers, nt, aux, data,
|
||||
splits[fold_idx], label_filter, device,
|
||||
)
|
||||
|
||||
if arr.size:
|
||||
means = arr.mean(axis=0)
|
||||
rows.append({
|
||||
"bridge": bridge_kind,
|
||||
"backbone": backbone,
|
||||
"rep": rep_idx,
|
||||
"fold": fold_idx,
|
||||
"n_samples": arr.shape[0],
|
||||
"img_value": float(means[0]),
|
||||
"cd_value": float(means[1]),
|
||||
"img_share": float(means[0] / (means.sum() + 1e-8)),
|
||||
"img_aux_auc": head_aucs.get("img_aux", float("nan")),
|
||||
"cd_aux_auc": head_aucs.get("cd_aux", float("nan")),
|
||||
"nt_head_auc": head_aucs.get("nt_head", float("nan")),
|
||||
})
|
||||
|
||||
del towers, nt, aux
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument(
|
||||
"--bridges", default="gated,ortho",
|
||||
help="Comma-separated bridge kinds to process (default: gated,ortho)",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--reps", type=int, default=None,
|
||||
help="Cap reps per backbone for a quick first pass (default: all)",
|
||||
)
|
||||
args = ap.parse_args()
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
print(f"device: {device}")
|
||||
|
||||
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
for bridge in args.bridges.split(","):
|
||||
bridge = bridge.strip()
|
||||
if not bridge:
|
||||
continue
|
||||
df = run_for_bridge(bridge, args.reps, device)
|
||||
suffix = "gates" if bridge == "gated" else "stream_norms"
|
||||
out = OUT_DIR / f"{bridge}_{suffix}.csv"
|
||||
df.to_csv(out, index=False)
|
||||
print(f"saved {out} ({len(df)} rows)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,10 @@
|
||||
[
|
||||
{
|
||||
"_note": "F2 block 2 — Anonymous CV variant of the refugelike R50 image-only single-eye baseline. split_identity_level=0 disables patient-level fold grouping (v4_hypertower.py:254-256); fold assignment becomes patient-anonymous, allowing the two eyes of one patient to fall on opposite sides of the train/test split. Mirrors v3 phase 2 imageonly_resnet50_leaky. Same backbone, same architecture, same seed/fold_seed as the F2 block-3 baseline (refugelike img-only single-eye); only the grouping rule changes. The gap baseline→anonymous quantifies the patient-anonymous CV inflation.",
|
||||
"run_name": "experiments/backbone_replication/anonymous_cv_refugelike",
|
||||
"reps": 10,
|
||||
"overrides": {
|
||||
"split_identity_level": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,10 @@
|
||||
[
|
||||
{
|
||||
"_note": "Anonymous CV variant of the single-eye img+cd L1 fusion configuration at the refugelike R50 backbone (the 'classic hypertower' eye-level fusion mode without L2 bilateral aggregation). split_identity_level=0 disables patient-level fold grouping; fold assignment becomes patient-anonymous, allowing the two eyes of one patient to fall on opposite sides of the train/test split. Mirrors the image-only anonymous_cv_refugelike run (F2 block 2) but with the clinical tower fused in at the L1 stage, so the patient-anonymous CV inflation can be quantified directly against the single-eye img+cd headline number (ensemble_single_refugelike rep-mean test AUC 0.874 at split_identity_level=1). Same backbone, towers, bridge, training schedule, and seeds as the single-eye headline; only the fold-grouping rule changes. Bilateral L2 is not included because patient-level aggregation is mechanically incompatible with patient-anonymous fold assignment.",
|
||||
"run_name": "experiments/backbone_replication/anonymous_cv_ensemble_single_refugelike",
|
||||
"reps": 10,
|
||||
"overrides": {
|
||||
"split_identity_level": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,46 @@
|
||||
[
|
||||
{
|
||||
"_note": "F2 block 1 — basic backbones, ImageNet pretrained, image-only, single-eye, patient-grouped 5-fold CV. VGG16. Replicates v3 phase 1 imageonly_vgg16 in the v4 pipeline.",
|
||||
"run_name": "experiments/backbone_replication/basic_vgg16",
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": { "args": { "backbone": "vgg16", "freeze_ratio": 0.0 } }
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"_note": "F2 block 1 — MobileNetV2 (ImageNet, img-only, single-eye).",
|
||||
"run_name": "experiments/backbone_replication/basic_mobilenet_v2",
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": { "args": { "backbone": "mobilenet_v2", "freeze_ratio": 0.0 } }
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"_note": "F2 block 1 — DenseNet121 (ImageNet, img-only, single-eye).",
|
||||
"run_name": "experiments/backbone_replication/basic_densenet121",
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": { "args": { "backbone": "densenet121", "freeze_ratio": 0.0 } }
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"_note": "F2 block 1 — InceptionV3 (ImageNet, img-only, single-eye). NB: Inception expects 299x299 input — backbone_transform_config auto-overrides crop_size to 299 for inception_v3.",
|
||||
"run_name": "experiments/backbone_replication/basic_inception_v3",
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": { "args": { "backbone": "inception_v3", "freeze_ratio": 0.0 } }
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"_note": "F2 block 1 — ResNet50 ImageNet-pretrained (the unfrozen ImageNet R50, not refugelike). Anchor for the basic-backbone comparison.",
|
||||
"run_name": "experiments/backbone_replication/basic_resnet50",
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": { "args": { "backbone": "resnet50", "freeze_ratio": 0.0 } }
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,39 @@
|
||||
[
|
||||
{
|
||||
"_note": "F2 block 2 — GT disc-contour crop. R50 refugelike, image-only, single-eye, patient-grouped 5-fold CV. Each input image is cropped to a square bbox centred on the GT disc contour with margin=2.5 (matching v3 phase 2 imageonly_resnet50_gtcrop_2.5). The crop happens in original image coords before the standard 256-resize + 224-center-crop transform pipeline runs. Eyes with no contour file fall back to the un-cropped full image.",
|
||||
"run_name": "experiments/backbone_replication/gtcrop_refugelike",
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": {
|
||||
"args": {
|
||||
"backbone": "refugelike",
|
||||
"freeze_ratio": 0.0,
|
||||
"crop_source": "gt",
|
||||
"crop_kwargs": { "margin": 2.5, "expert": 1 }
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"_note": "F2 block 2 — U-Net disc-mask crop. R50 refugelike, image-only, single-eye, patient-grouped 5-fold CV. Each input image is cropped to a square bbox centred on the U-Net-predicted disc mask with margin=2.5 (matching v3 phase 2 imageonly_resnet50_unetcrop_2.5). U-Net is loaded from the base REFUGE checkpoint and NOT fine-tuned per fold (finetune_epochs=0) to keep this an apples-to-apples preprocessing-only ablation. Eyes where U-Net predicts no disc fall back to the un-cropped full image.",
|
||||
"run_name": "experiments/backbone_replication/unetcrop_refugelike",
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": {
|
||||
"args": {
|
||||
"backbone": "refugelike",
|
||||
"freeze_ratio": 0.0,
|
||||
"crop_source": "unet",
|
||||
"crop_kwargs": {
|
||||
"margin": 2.5,
|
||||
"weights_path": "models/v2/refuge/segmentation/per_image/best.pt",
|
||||
"unet_size": 512,
|
||||
"threshold": 0.5,
|
||||
"finetune_epochs": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,10 @@
|
||||
[
|
||||
{
|
||||
"_note": "10-rep checkpointed run of the production bilateral img+cd ensemble at the refugelike R50 backbone (headline configuration). Per-fold tower and stage_models state_dicts saved under each rep's checkpoints/foldN/ directory. Used for F8 explainability: reconstructing per-tower predictions at the nt (eye) and hb (patient) levels so we can compare img-tower-only, cd-tower-only, eye-fusion, and patient-fusion outputs on the headline backbone.",
|
||||
"run_name": "experiments/explainability/ensemble_refugelike_ckpt",
|
||||
"reps": 10,
|
||||
"overrides": {
|
||||
"save_checkpoints": true
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -1,8 +1,8 @@
|
||||
[
|
||||
{
|
||||
"_note": "refugelike (resnet50 + REFUGE fundus pretraining), freeze stem only (1/5 blocks). Tests whether even minimal anchoring helps stability.",
|
||||
"_note": "refugelike (resnet50 + REFUGE fundus pretraining), freeze stem only (1/5 blocks). Tests whether even minimal anchoring helps stability. Bumped to 10 reps for the methods ablation table.",
|
||||
"run_name": "experiments/freeze_sweep/refugelike_freeze20",
|
||||
"reps": 3,
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": { "args": { "backbone": "refugelike", "freeze_ratio": 0.2 } }
|
||||
}
|
||||
@@ -18,11 +18,20 @@
|
||||
},
|
||||
|
||||
{
|
||||
"_note": "Freeze stem + layer1 + layer2 (3/5 blocks). Only the deep semantic layers adapt — most aggressive practical setting before model loses capacity.",
|
||||
"_note": "Freeze stem + layer1 + layer2 (3/5 blocks). Only the deep semantic layers adapt — most aggressive practical setting before model loses capacity. Bumped to 10 reps for the methods ablation table.",
|
||||
"run_name": "experiments/freeze_sweep/refugelike_freeze60",
|
||||
"reps": 3,
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": { "args": { "backbone": "refugelike", "freeze_ratio": 0.6 } }
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"_note": "Fully frozen backbone (freeze_ratio=1.0) — image features are entirely fixed at the REFUGE-pretrained state, only the L1 bridge and downstream heads adapt. Provides the extreme end of the freeze-ratio sweep for the methods ablation table.",
|
||||
"run_name": "experiments/freeze_sweep/refugelike_freeze100",
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": { "args": { "backbone": "refugelike", "freeze_ratio": 1.0 } }
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
[
|
||||
{
|
||||
"_note": "Re-run of the headline VF_MD regression configuration (baseline_reg_nt50) after the PredictionStore y_true dtype fix. The previous run stored y_true as int64, silently rounding VF_MD floats to integers (introducing ~0.3 dB of rounding noise into MAE and the regression residuals). With the fix applied, regression targets are stored as float64. Matched seeds and fold_seeds to the original baseline_reg_nt50 so fold splits are identical and rep-paired comparison is meaningful. All other settings (nt epochs = 50, all four heads as RegressionHead targeting vf_md, label_filter [0,1,2] to retain Suspect, refugelike R50 backbone, bilateral L1+L2 fusion) are unchanged.",
|
||||
"run_name": "experiments/reg_head/baseline_reg_nt50_floaty",
|
||||
"reps": 10,
|
||||
"overrides": {
|
||||
"label_filter": [0, 1, 2]
|
||||
},
|
||||
"stage_overrides": {
|
||||
"nt": { "epochs": 50 },
|
||||
"img_aux": {
|
||||
"module": "v4.classes.heads.regression",
|
||||
"class": "RegressionHead",
|
||||
"args": { "dropout": 0.3, "target_key": "vf_md", "loss": "mse" }
|
||||
},
|
||||
"cd_aux": {
|
||||
"module": "v4.classes.heads.regression",
|
||||
"class": "RegressionHead",
|
||||
"args": { "dropout": 0.3, "target_key": "vf_md", "loss": "mse" }
|
||||
},
|
||||
"nt_head": {
|
||||
"module": "v4.classes.heads.regression",
|
||||
"class": "RegressionHead",
|
||||
"args": { "dropout": 0.3, "target_key": "vf_md", "loss": "mse" }
|
||||
},
|
||||
"hb_head": {
|
||||
"module": "v4.classes.heads.regression",
|
||||
"class": "RegressionHead",
|
||||
"args": { "dropout": 0.3, "target_key": "vf_md", "loss": "mse" }
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,16 @@
|
||||
[
|
||||
{
|
||||
"_note": "Sensitivity: refugelike ensemble (img + cd, bilateral) — apples-to-apples replication of tri_v1/baseline_ensemble (mean hb_test_auc ≈ 0.896) but with Axial_Length INCLUDED in the clinical feature set instead of excluded. Tests whether the pilot-era finding that Axial_Length negatively contributed to fused AUC survives the v4 architecture. Same seed/fold_seed start (1234/100) as baseline_ensemble so (rep, fold) pairs are matched for paired statistics. save_checkpoints + save_predictions enabled so the run can drive a downstream permutation-importance feature ablation if needed.",
|
||||
"run_name": "experiments/sensitivity/refugelike_ensemble_with_axial_length",
|
||||
"reps": 10,
|
||||
"overrides": {
|
||||
"save_checkpoints": true,
|
||||
"save_predictions": true,
|
||||
"data": {
|
||||
"args": {
|
||||
"exclude_cols": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Memory probe: EfficientNetV2-M at 480x480, bilateral forward+backward, AMP bf16.
|
||||
|
||||
Goal: confirm bs=8 fits in 16 GB on the available GPU before committing to the
|
||||
full sensitivity experiment.
|
||||
|
||||
Mimics the bilateral training step (image tower run twice on OD + OS with shared
|
||||
weights, plus a small downstream head + CE loss + Adam step). The clinical tower
|
||||
and L1 bridge are omitted; their memory footprint is negligible against V2-M
|
||||
activations. Synthetic inputs of the correct shape — no v4 dataset needed.
|
||||
|
||||
Run: python v4/scripts/experiments/sensitivity/probe_v2m_480_amp.py
|
||||
Expected output: GPU name, peak memory at each phase, fit/OOM verdict.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import torch
|
||||
from torch import nn
|
||||
from torchvision.models import efficientnet_v2_m
|
||||
|
||||
|
||||
BATCH = 8
|
||||
RES = 480
|
||||
DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
def fmt_gb(bytes_):
|
||||
return f"{bytes_ / 1024**3:.2f} GB"
|
||||
|
||||
|
||||
def main():
|
||||
if not torch.cuda.is_available():
|
||||
print("No CUDA/ROCm device available; probe requires a GPU.", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
device = torch.device("cuda")
|
||||
gpu_name = torch.cuda.get_device_name(0)
|
||||
gpu_total = torch.cuda.get_device_properties(0).total_memory
|
||||
print(f"GPU: {gpu_name} total VRAM: {fmt_gb(gpu_total)}")
|
||||
print(f"Config: V2-M, bs={BATCH}, res={RES}, bilateral 2x forward, AMP={DTYPE}")
|
||||
print("-" * 70)
|
||||
|
||||
torch.cuda.empty_cache()
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
|
||||
backbone = efficientnet_v2_m(weights=None).to(device)
|
||||
feat_dim = backbone.classifier[1].in_features
|
||||
backbone.classifier = nn.Identity()
|
||||
head = nn.Sequential(
|
||||
nn.LayerNorm(feat_dim),
|
||||
nn.Linear(feat_dim, 256),
|
||||
nn.GELU(),
|
||||
nn.Linear(256, 2),
|
||||
).to(device)
|
||||
opt = torch.optim.Adam(
|
||||
list(backbone.parameters()) + list(head.parameters()),
|
||||
lr=1e-4,
|
||||
)
|
||||
|
||||
print(f"After model + optimizer load: allocated={fmt_gb(torch.cuda.memory_allocated())} "
|
||||
f"peak={fmt_gb(torch.cuda.max_memory_allocated())}")
|
||||
|
||||
x_od = torch.randn(BATCH, 3, RES, RES, device=device)
|
||||
x_os = torch.randn(BATCH, 3, RES, RES, device=device)
|
||||
y = torch.randint(0, 2, (BATCH,), device=device)
|
||||
|
||||
print(f"After synthetic inputs: allocated={fmt_gb(torch.cuda.memory_allocated())} "
|
||||
f"peak={fmt_gb(torch.cuda.max_memory_allocated())}")
|
||||
|
||||
try:
|
||||
opt.zero_grad(set_to_none=True)
|
||||
with torch.autocast(device_type="cuda", dtype=DTYPE):
|
||||
z_od = backbone(x_od)
|
||||
z_os = backbone(x_os)
|
||||
z = z_od + z_os
|
||||
logits = head(z)
|
||||
loss = nn.functional.cross_entropy(logits, y)
|
||||
|
||||
print(f"After forward: allocated={fmt_gb(torch.cuda.memory_allocated())} "
|
||||
f"peak={fmt_gb(torch.cuda.max_memory_allocated())}")
|
||||
|
||||
loss.backward()
|
||||
print(f"After backward: allocated={fmt_gb(torch.cuda.memory_allocated())} "
|
||||
f"peak={fmt_gb(torch.cuda.max_memory_allocated())}")
|
||||
|
||||
opt.step()
|
||||
print(f"After optimizer step: allocated={fmt_gb(torch.cuda.memory_allocated())} "
|
||||
f"peak={fmt_gb(torch.cuda.max_memory_allocated())}")
|
||||
|
||||
torch.cuda.synchronize()
|
||||
peak = torch.cuda.max_memory_allocated()
|
||||
headroom = gpu_total - peak
|
||||
print("-" * 70)
|
||||
print(f"VERDICT: FIT | peak={fmt_gb(peak)} of {fmt_gb(gpu_total)} "
|
||||
f"headroom={fmt_gb(headroom)} ({100 * headroom / gpu_total:.1f}%)")
|
||||
print(f"Loss value: {loss.item():.4f}")
|
||||
|
||||
except torch.cuda.OutOfMemoryError as e:
|
||||
peak = torch.cuda.max_memory_allocated()
|
||||
print("-" * 70)
|
||||
print(f"VERDICT: OOM | peak before OOM={fmt_gb(peak)} of {fmt_gb(gpu_total)}")
|
||||
print(f"OOM details: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,24 @@
|
||||
[
|
||||
{
|
||||
"_note": "Sensitivity: refuge V2-M ensemble (img + cd, bilateral, ortho-w0.1 inner-Hadamard not applied here — matches the plain V2-M ensemble baseline at 224 from efficientnet/refuge_efficientnetv2_m which gave 0.9132 ± 0.019). This run feeds EfficientNetV2-M at its NATIVE 480x480 input resolution instead of the pipeline default 224. Activation memory roughly 4.6x; bf16 autocast keeps bs=8 fit in 16 GB (probe peak 9.07 GB on 7800 XT). Matched seed/fold_seed (1234/100) inherited from ensemble_fused.json so reps 1-10 here pair with reps 1-10 of the 224 baseline for paired statistics. 10 reps queued — kill early if wall-clock proves prohibitive. save_checkpoints + save_predictions enabled for downstream analysis if the result is promising.",
|
||||
"run_name": "experiments/sensitivity/v2m_at_480_amp",
|
||||
"reps": 10,
|
||||
"overrides": {
|
||||
"save_checkpoints": true,
|
||||
"save_predictions": true,
|
||||
"training": {
|
||||
"amp": true,
|
||||
"amp_dtype": "bfloat16"
|
||||
}
|
||||
},
|
||||
"tower_overrides": {
|
||||
"img": {
|
||||
"args": {
|
||||
"backbone": "refuge_efficientnet_v2_m",
|
||||
"freeze_ratio": 0.0,
|
||||
"crop_size": 480
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user