Add analysis scripts and experiment configurations for bridge attention and sensitivity studies
- Introduced `bridge_attention_ceiling_check.py` for variance decomposition analysis on bridge attention configurations. - Added `bridge_attention_readout.py` to perform per-tower gate and contribution readouts, including AUC sanity checks. - Created multiple JSON configuration files for backbone replication experiments, including anonymous CV variants and basic backbones. - Implemented sensitivity experiments to evaluate the impact of axial length inclusion and EfficientNetV2-M performance at higher resolutions. - Added a memory probe script to assess GPU memory usage during training with EfficientNetV2-M.
This commit is contained in:
rpotter6298
@@ -0,0 +1,10 @@
|
||||
[
|
||||
{
|
||||
"_note": "F2 block 2 — Anonymous CV variant of the refugelike R50 image-only single-eye baseline. split_identity_level=0 disables patient-level fold grouping (v4_hypertower.py:254-256); fold assignment becomes patient-anonymous, allowing the two eyes of one patient to fall on opposite sides of the train/test split. Mirrors v3 phase 2 imageonly_resnet50_leaky. Same backbone, same architecture, same seed/fold_seed as the F2 block-3 baseline (refugelike img-only single-eye); only the grouping rule changes. The gap baseline→anonymous quantifies the patient-anonymous CV inflation.",
|
||||
"run_name": "experiments/backbone_replication/anonymous_cv_refugelike",
|
||||
"reps": 10,
|
||||
"overrides": {
|
||||
"split_identity_level": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,10 @@
|
||||
[
|
||||
{
|
||||
"_note": "Anonymous CV variant of the single-eye img+cd L1 fusion configuration at the refugelike R50 backbone (the 'classic hypertower' eye-level fusion mode without L2 bilateral aggregation). split_identity_level=0 disables patient-level fold grouping; fold assignment becomes patient-anonymous, allowing the two eyes of one patient to fall on opposite sides of the train/test split. Mirrors the image-only anonymous_cv_refugelike run (F2 block 2) but with the clinical tower fused in at the L1 stage, so the patient-anonymous CV inflation can be quantified directly against the single-eye img+cd headline number (ensemble_single_refugelike rep-mean test AUC 0.874 at split_identity_level=1). Same backbone, towers, bridge, training schedule, and seeds as the single-eye headline; only the fold-grouping rule changes. Bilateral L2 is not included because patient-level aggregation is mechanically incompatible with patient-anonymous fold assignment.",
|
||||
"run_name": "experiments/backbone_replication/anonymous_cv_ensemble_single_refugelike",
|
||||
"reps": 10,
|
||||
"overrides": {
|
||||
"split_identity_level": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,46 @@
|
||||
[
|
||||
{
|
||||
"_note": "F2 block 1 — basic backbones, ImageNet pretrained, image-only, single-eye, patient-grouped 5-fold CV. VGG16. Replicates v3 phase 1 imageonly_vgg16 in the v4 pipeline.",
|
||||
"run_name": "experiments/backbone_replication/basic_vgg16",
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": { "args": { "backbone": "vgg16", "freeze_ratio": 0.0 } }
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"_note": "F2 block 1 — MobileNetV2 (ImageNet, img-only, single-eye).",
|
||||
"run_name": "experiments/backbone_replication/basic_mobilenet_v2",
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": { "args": { "backbone": "mobilenet_v2", "freeze_ratio": 0.0 } }
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"_note": "F2 block 1 — DenseNet121 (ImageNet, img-only, single-eye).",
|
||||
"run_name": "experiments/backbone_replication/basic_densenet121",
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": { "args": { "backbone": "densenet121", "freeze_ratio": 0.0 } }
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"_note": "F2 block 1 — InceptionV3 (ImageNet, img-only, single-eye). NB: Inception expects 299x299 input — backbone_transform_config auto-overrides crop_size to 299 for inception_v3.",
|
||||
"run_name": "experiments/backbone_replication/basic_inception_v3",
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": { "args": { "backbone": "inception_v3", "freeze_ratio": 0.0 } }
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"_note": "F2 block 1 — ResNet50 ImageNet-pretrained (the unfrozen ImageNet R50, not refugelike). Anchor for the basic-backbone comparison.",
|
||||
"run_name": "experiments/backbone_replication/basic_resnet50",
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": { "args": { "backbone": "resnet50", "freeze_ratio": 0.0 } }
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,39 @@
|
||||
[
|
||||
{
|
||||
"_note": "F2 block 2 — GT disc-contour crop. R50 refugelike, image-only, single-eye, patient-grouped 5-fold CV. Each input image is cropped to a square bbox centred on the GT disc contour with margin=2.5 (matching v3 phase 2 imageonly_resnet50_gtcrop_2.5). The crop happens in original image coords before the standard 256-resize + 224-center-crop transform pipeline runs. Eyes with no contour file fall back to the un-cropped full image.",
|
||||
"run_name": "experiments/backbone_replication/gtcrop_refugelike",
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": {
|
||||
"args": {
|
||||
"backbone": "refugelike",
|
||||
"freeze_ratio": 0.0,
|
||||
"crop_source": "gt",
|
||||
"crop_kwargs": { "margin": 2.5, "expert": 1 }
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"_note": "F2 block 2 — U-Net disc-mask crop. R50 refugelike, image-only, single-eye, patient-grouped 5-fold CV. Each input image is cropped to a square bbox centred on the U-Net-predicted disc mask with margin=2.5 (matching v3 phase 2 imageonly_resnet50_unetcrop_2.5). U-Net is loaded from the base REFUGE checkpoint and NOT fine-tuned per fold (finetune_epochs=0) to keep this an apples-to-apples preprocessing-only ablation. Eyes where U-Net predicts no disc fall back to the un-cropped full image.",
|
||||
"run_name": "experiments/backbone_replication/unetcrop_refugelike",
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": {
|
||||
"args": {
|
||||
"backbone": "refugelike",
|
||||
"freeze_ratio": 0.0,
|
||||
"crop_source": "unet",
|
||||
"crop_kwargs": {
|
||||
"margin": 2.5,
|
||||
"weights_path": "models/v2/refuge/segmentation/per_image/best.pt",
|
||||
"unet_size": 512,
|
||||
"threshold": 0.5,
|
||||
"finetune_epochs": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,10 @@
|
||||
[
|
||||
{
|
||||
"_note": "10-rep checkpointed run of the production bilateral img+cd ensemble at the refugelike R50 backbone (headline configuration). Per-fold tower and stage_models state_dicts saved under each rep's checkpoints/foldN/ directory. Used for F8 explainability: reconstructing per-tower predictions at the nt (eye) and hb (patient) levels so we can compare img-tower-only, cd-tower-only, eye-fusion, and patient-fusion outputs on the headline backbone.",
|
||||
"run_name": "experiments/explainability/ensemble_refugelike_ckpt",
|
||||
"reps": 10,
|
||||
"overrides": {
|
||||
"save_checkpoints": true
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -1,8 +1,8 @@
|
||||
[
|
||||
{
|
||||
"_note": "refugelike (resnet50 + REFUGE fundus pretraining), freeze stem only (1/5 blocks). Tests whether even minimal anchoring helps stability.",
|
||||
"_note": "refugelike (resnet50 + REFUGE fundus pretraining), freeze stem only (1/5 blocks). Tests whether even minimal anchoring helps stability. Bumped to 10 reps for the methods ablation table.",
|
||||
"run_name": "experiments/freeze_sweep/refugelike_freeze20",
|
||||
"reps": 3,
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": { "args": { "backbone": "refugelike", "freeze_ratio": 0.2 } }
|
||||
}
|
||||
@@ -18,11 +18,20 @@
|
||||
},
|
||||
|
||||
{
|
||||
"_note": "Freeze stem + layer1 + layer2 (3/5 blocks). Only the deep semantic layers adapt — most aggressive practical setting before model loses capacity.",
|
||||
"_note": "Freeze stem + layer1 + layer2 (3/5 blocks). Only the deep semantic layers adapt — most aggressive practical setting before model loses capacity. Bumped to 10 reps for the methods ablation table.",
|
||||
"run_name": "experiments/freeze_sweep/refugelike_freeze60",
|
||||
"reps": 3,
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": { "args": { "backbone": "refugelike", "freeze_ratio": 0.6 } }
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"_note": "Fully frozen backbone (freeze_ratio=1.0) — image features are entirely fixed at the REFUGE-pretrained state, only the L1 bridge and downstream heads adapt. Provides the extreme end of the freeze-ratio sweep for the methods ablation table.",
|
||||
"run_name": "experiments/freeze_sweep/refugelike_freeze100",
|
||||
"reps": 10,
|
||||
"tower_overrides": {
|
||||
"img": { "args": { "backbone": "refugelike", "freeze_ratio": 1.0 } }
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
[
|
||||
{
|
||||
"_note": "Re-run of the headline VF_MD regression configuration (baseline_reg_nt50) after the PredictionStore y_true dtype fix. The previous run stored y_true as int64, silently rounding VF_MD floats to integers (introducing ~0.3 dB of rounding noise into MAE and the regression residuals). With the fix applied, regression targets are stored as float64. Matched seeds and fold_seeds to the original baseline_reg_nt50 so fold splits are identical and rep-paired comparison is meaningful. All other settings (nt epochs = 50, all four heads as RegressionHead targeting vf_md, label_filter [0,1,2] to retain Suspect, refugelike R50 backbone, bilateral L1+L2 fusion) are unchanged.",
|
||||
"run_name": "experiments/reg_head/baseline_reg_nt50_floaty",
|
||||
"reps": 10,
|
||||
"overrides": {
|
||||
"label_filter": [0, 1, 2]
|
||||
},
|
||||
"stage_overrides": {
|
||||
"nt": { "epochs": 50 },
|
||||
"img_aux": {
|
||||
"module": "v4.classes.heads.regression",
|
||||
"class": "RegressionHead",
|
||||
"args": { "dropout": 0.3, "target_key": "vf_md", "loss": "mse" }
|
||||
},
|
||||
"cd_aux": {
|
||||
"module": "v4.classes.heads.regression",
|
||||
"class": "RegressionHead",
|
||||
"args": { "dropout": 0.3, "target_key": "vf_md", "loss": "mse" }
|
||||
},
|
||||
"nt_head": {
|
||||
"module": "v4.classes.heads.regression",
|
||||
"class": "RegressionHead",
|
||||
"args": { "dropout": 0.3, "target_key": "vf_md", "loss": "mse" }
|
||||
},
|
||||
"hb_head": {
|
||||
"module": "v4.classes.heads.regression",
|
||||
"class": "RegressionHead",
|
||||
"args": { "dropout": 0.3, "target_key": "vf_md", "loss": "mse" }
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,16 @@
|
||||
[
|
||||
{
|
||||
"_note": "Sensitivity: refugelike ensemble (img + cd, bilateral) — apples-to-apples replication of tri_v1/baseline_ensemble (mean hb_test_auc ≈ 0.896) but with Axial_Length INCLUDED in the clinical feature set instead of excluded. Tests whether the pilot-era finding that Axial_Length negatively contributed to fused AUC survives the v4 architecture. Same seed/fold_seed start (1234/100) as baseline_ensemble so (rep, fold) pairs are matched for paired statistics. save_checkpoints + save_predictions enabled so the run can drive a downstream permutation-importance feature ablation if needed.",
|
||||
"run_name": "experiments/sensitivity/refugelike_ensemble_with_axial_length",
|
||||
"reps": 10,
|
||||
"overrides": {
|
||||
"save_checkpoints": true,
|
||||
"save_predictions": true,
|
||||
"data": {
|
||||
"args": {
|
||||
"exclude_cols": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Memory probe: EfficientNetV2-M at 480x480, bilateral forward+backward, AMP bf16.
|
||||
|
||||
Goal: confirm bs=8 fits in 16 GB on the available GPU before committing to the
|
||||
full sensitivity experiment.
|
||||
|
||||
Mimics the bilateral training step (image tower run twice on OD + OS with shared
|
||||
weights, plus a small downstream head + CE loss + Adam step). The clinical tower
|
||||
and L1 bridge are omitted; their memory footprint is negligible against V2-M
|
||||
activations. Synthetic inputs of the correct shape — no v4 dataset needed.
|
||||
|
||||
Run: python v4/scripts/experiments/sensitivity/probe_v2m_480_amp.py
|
||||
Expected output: GPU name, peak memory at each phase, fit/OOM verdict.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import torch
|
||||
from torch import nn
|
||||
from torchvision.models import efficientnet_v2_m
|
||||
|
||||
|
||||
BATCH = 8
|
||||
RES = 480
|
||||
DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
def fmt_gb(bytes_):
|
||||
return f"{bytes_ / 1024**3:.2f} GB"
|
||||
|
||||
|
||||
def main():
|
||||
if not torch.cuda.is_available():
|
||||
print("No CUDA/ROCm device available; probe requires a GPU.", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
device = torch.device("cuda")
|
||||
gpu_name = torch.cuda.get_device_name(0)
|
||||
gpu_total = torch.cuda.get_device_properties(0).total_memory
|
||||
print(f"GPU: {gpu_name} total VRAM: {fmt_gb(gpu_total)}")
|
||||
print(f"Config: V2-M, bs={BATCH}, res={RES}, bilateral 2x forward, AMP={DTYPE}")
|
||||
print("-" * 70)
|
||||
|
||||
torch.cuda.empty_cache()
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
|
||||
backbone = efficientnet_v2_m(weights=None).to(device)
|
||||
feat_dim = backbone.classifier[1].in_features
|
||||
backbone.classifier = nn.Identity()
|
||||
head = nn.Sequential(
|
||||
nn.LayerNorm(feat_dim),
|
||||
nn.Linear(feat_dim, 256),
|
||||
nn.GELU(),
|
||||
nn.Linear(256, 2),
|
||||
).to(device)
|
||||
opt = torch.optim.Adam(
|
||||
list(backbone.parameters()) + list(head.parameters()),
|
||||
lr=1e-4,
|
||||
)
|
||||
|
||||
print(f"After model + optimizer load: allocated={fmt_gb(torch.cuda.memory_allocated())} "
|
||||
f"peak={fmt_gb(torch.cuda.max_memory_allocated())}")
|
||||
|
||||
x_od = torch.randn(BATCH, 3, RES, RES, device=device)
|
||||
x_os = torch.randn(BATCH, 3, RES, RES, device=device)
|
||||
y = torch.randint(0, 2, (BATCH,), device=device)
|
||||
|
||||
print(f"After synthetic inputs: allocated={fmt_gb(torch.cuda.memory_allocated())} "
|
||||
f"peak={fmt_gb(torch.cuda.max_memory_allocated())}")
|
||||
|
||||
try:
|
||||
opt.zero_grad(set_to_none=True)
|
||||
with torch.autocast(device_type="cuda", dtype=DTYPE):
|
||||
z_od = backbone(x_od)
|
||||
z_os = backbone(x_os)
|
||||
z = z_od + z_os
|
||||
logits = head(z)
|
||||
loss = nn.functional.cross_entropy(logits, y)
|
||||
|
||||
print(f"After forward: allocated={fmt_gb(torch.cuda.memory_allocated())} "
|
||||
f"peak={fmt_gb(torch.cuda.max_memory_allocated())}")
|
||||
|
||||
loss.backward()
|
||||
print(f"After backward: allocated={fmt_gb(torch.cuda.memory_allocated())} "
|
||||
f"peak={fmt_gb(torch.cuda.max_memory_allocated())}")
|
||||
|
||||
opt.step()
|
||||
print(f"After optimizer step: allocated={fmt_gb(torch.cuda.memory_allocated())} "
|
||||
f"peak={fmt_gb(torch.cuda.max_memory_allocated())}")
|
||||
|
||||
torch.cuda.synchronize()
|
||||
peak = torch.cuda.max_memory_allocated()
|
||||
headroom = gpu_total - peak
|
||||
print("-" * 70)
|
||||
print(f"VERDICT: FIT | peak={fmt_gb(peak)} of {fmt_gb(gpu_total)} "
|
||||
f"headroom={fmt_gb(headroom)} ({100 * headroom / gpu_total:.1f}%)")
|
||||
print(f"Loss value: {loss.item():.4f}")
|
||||
|
||||
except torch.cuda.OutOfMemoryError as e:
|
||||
peak = torch.cuda.max_memory_allocated()
|
||||
print("-" * 70)
|
||||
print(f"VERDICT: OOM | peak before OOM={fmt_gb(peak)} of {fmt_gb(gpu_total)}")
|
||||
print(f"OOM details: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,24 @@
|
||||
[
|
||||
{
|
||||
"_note": "Sensitivity: refuge V2-M ensemble (img + cd, bilateral, ortho-w0.1 inner-Hadamard not applied here — matches the plain V2-M ensemble baseline at 224 from efficientnet/refuge_efficientnetv2_m which gave 0.9132 ± 0.019). This run feeds EfficientNetV2-M at its NATIVE 480x480 input resolution instead of the pipeline default 224. Activation memory roughly 4.6x; bf16 autocast keeps bs=8 fit in 16 GB (probe peak 9.07 GB on 7800 XT). Matched seed/fold_seed (1234/100) inherited from ensemble_fused.json so reps 1-10 here pair with reps 1-10 of the 224 baseline for paired statistics. 10 reps queued — kill early if wall-clock proves prohibitive. save_checkpoints + save_predictions enabled for downstream analysis if the result is promising.",
|
||||
"run_name": "experiments/sensitivity/v2m_at_480_amp",
|
||||
"reps": 10,
|
||||
"overrides": {
|
||||
"save_checkpoints": true,
|
||||
"save_predictions": true,
|
||||
"training": {
|
||||
"amp": true,
|
||||
"amp_dtype": "bfloat16"
|
||||
}
|
||||
},
|
||||
"tower_overrides": {
|
||||
"img": {
|
||||
"args": {
|
||||
"backbone": "refuge_efficientnet_v2_m",
|
||||
"freeze_ratio": 0.0,
|
||||
"crop_size": 480
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user