ryan/hypertower
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: ryan/hypertower:d36b9508ff75a5188f3eb016d7d3aec02e17436a
Branches Tags
ryan/hypertower:main
...
compare: ryan/hypertower:af813bbb62d3b97a7056129ee68700fa22eade75
Branches Tags
ryan/hypertower:main

10 Commits
d36b9508ff ... af813bbb62

Author SHA1 Message Date
rpotter6298 af813bbb62 Refactor geometry feature loaders and enhance distributed client status reporting 
- Updated GTGeometryLoader to streamline geometry vector computation and caching.
- Introduced UNetGeometryLoader for UNet-derived geometry vectors.
- Added sample collection method in PapilaBundle for better data handling.
- Refined GeometrySegEncoder and ImageEncoder to utilize new sample collection.
- Enhanced distributed client with heartbeat mechanism for improved job tracking.
- Added new configuration files for UNet-derived geometry integration.
 2026-04-29 11:05:19 +02:00
rpotter6298 512ebd13b2 Add distributed server implementation and protocol definitions 
- Introduced `protocol.py` for shared data models used in server/client communication, including request and response schemas for registration, job submission, and status updates.
- Implemented `server.py` to manage a SQLite job queue and client registry, handling job polling, status updates, and job completion.
- Created a cheat sheet for server usage, detailing commands for starting the server, submitting jobs, and monitoring clients.
- Added several experiment configuration files for various training setups, including geometry vector injections and baseline ensembles.
 2026-04-28 08:24:25 +02:00
rpotter6298 4dea45df78 v4 update 2026-04-20 18:01:31 +02:00
rpotter6298 13290575d5 pre-refactor 041426 2026-04-14 19:42:16 +02:00
rpotter6298 eb9eafe715 began work on v3 2026-03-19 16:58:29 +01:00
rpotter6298 786457b30d update 3-19 2026-03-19 11:18:58 +01:00
rpotter6298 7ea85d5426 added better memory caching, multithreaded processing, cleanup scripts dir 2026-03-13 08:08:36 +01:00
rpotter6298 8282461a23 removed accidental upload 2026-03-12 16:20:21 +01:00
rpotter6298 6d0698d0fb reworked iop_corr, added explainability tools and plotting tools, cleanup codebase 2026-03-12 16:12:59 +01:00
rpotter6298 13ad32683f predictions class added 2026-03-04 11:17:10 +01:00
254 changed files with 36077 additions and 6035 deletions
Expand all files Collapse all files
.codex
View File
.gitignore
+9
View File
@@ -9,3 +9,12 @@ cache_data/
# model artifacts
models/refuge/
models/v2/refuge/
**/.archive/
.archive/
scripts/deprecated/
v3/results/*
scripts/utility/backup_mirror_with_archive.sh
v3/distributed/logs/*
v4/configs/**/
v4/distributed/logs/*
v4/results/*
TODO.md
+23
View File
@@ -0,0 +1,23 @@
# Project TODO
## Paper
- [ ] **Learning curve analysis** — train on 25/50/75/100% of training data, plot AUC vs n.
Motivation: empirical evidence that the model is data-starved, which justifies the decision
not to pursue attention-gating (transformer) extensions to the NTowerHT bridge.
If the curve is still ascending at full data → supports the argument that a more expressive
architecture would overfit at this sample size. Generates a figure for the paper.
- [ ] **GradCAM nasal-side analysis** — re-run GradCAM separately for OD and OS eyes rather
than aggregated. The current aggregation mirrors the two eyes against each other, washing out
any directional bias. Clinically, we would expect GradCAM attention offset from the disc center
to trend toward the nasal side (where RNFL loss presents earliest in glaucoma). If the model
has learned this, it would only be visible in per-side heatmaps — OD and OS are mirror images
so the nasal direction is opposite for each. This could be a strong interpretability result
for the paper if the bias is present.
- [ ] **Quantify attention-gating as future work** — use the learning curve result + parameter
count ratio (Q/K/V projections over fusion_dim vs training n) to formally justify the choice.
Frame in paper as: "we identify cross-attention inside the NTowerHT bridge as a promising
extension, but our sample size (N≈400 training patients) is insufficient to avoid overfitting
a more expressive interaction layer" — cite the learning curve figure as evidence.
classes/__init__.py
-19
View File
@@ -1,19 +0,0 @@
from .clinical_data import ClinicalData
from .dataset import ClinicalDataset
from .image_tower import ImageTower
from .md_tower import MDTower
from .bridge import Bridge, VoteBridge
# from .hypertower import HyperTower
from .backbones import list_names, BackboneSpec, BACKBONES
from .papila_builders import build_papila_clinical
from .SE_attention import SEBlock, SEGateLogger
from .early_stop import EarlyStopper
__all__ = [
"ClinicalData",
"ClinicalDataset",
"ImageTower",
"MDTower",
"Bridge",
"VoteBridge",
# "HyperTower",
]
classes/bridge.py
-105
View File
@@ -1,105 +0,0 @@
# bridge.py
import torch
import torch.nn as nn
from classes.SE_attention import SEBlock, SEGateLogger
# class SEBlock(nn.Module):
# def __init__(self, dim: int, reduction: int = 16):
# super().__init__()
# hidden = max(1, dim // max(1, reduction))
# self.net = nn.Sequential(
# nn.Linear(dim, hidden, bias=True),
# nn.ReLU(inplace=True),
# nn.Linear(hidden, dim, bias=True),
# nn.Sigmoid(),
# )
# def forward(self, x):
# return self.net(x)
class Bridge(nn.Module):
def __init__(
self,
img_dim,
meta_dim,
num_classes,
fusion_dim=256,
mode="fused",
use_se: bool = True,
se_reduction: int = 16,
se_pre_norm: bool = True,
):
super().__init__()
self.mode = mode
self.use_se = use_se
# self.se_reduction = se_reduction
# self.se_pre_norm = se_pre_norm
#project towers to equal width
self.W_img = nn.Linear(img_dim, fusion_dim)
self.W_md = nn.Linear(meta_dim, fusion_dim)
#(optional) : set layernorm for se so one tower doesn't dominate the other
self.ln_img = nn.LayerNorm(fusion_dim) if se_pre_norm else nn.Identity()
self.ln_md = nn.LayerNorm(fusion_dim) if se_pre_norm else nn.Identity()
#SE gate on the fused vector
self.se = SEBlock(fusion_dim, reduction=se_reduction, residual=True) if use_se else None
self.se_log = SEGateLogger(enabled=use_se, track_channels=False, dim=fusion_dim)
#heads
self.classifier_fused = nn.Sequential(
nn.ReLU(), nn.Dropout(0.5), nn.Linear(fusion_dim, num_classes)
)
self.classifier_img = nn.Linear(img_dim, num_classes)
self.classifier_md = nn.Linear(meta_dim, num_classes)
def reset_se_stats(self):
"""Call at epoch start."""
if getattr(self, "se_log", None):
self.se_log.reset()
def get_se_stats(self, reset: bool = True):
"""Call after eval. Returns dict or None."""
if getattr(self, "se_log", None) and self.se_log.enabled:
return self.se_log.get(reset=reset)
return None
def forward(self, img_feats, md_feats):
out_img = None if self.mode == "metadata_only" else self.classifier_img(img_feats)
out_md = None if self.mode == "image_only" else self.classifier_md(md_feats)
if self.mode == "fused":
hi = self.ln_img(self.W_img(img_feats)) #image features
hm = self.ln_md(self.W_md(md_feats)) #metadata features
fused = hi * hm #elementwise product
#apply SE gates
if self.se is not None:
fused, gates = self.se(fused)
if self.se_log.enabled:
self.se_log.accumulate(gates)
if self.se is not None and self.training and self.se_log.enabled:
if not hasattr(self, "_dbg_seen"):
self._dbg_seen = 0
if self._dbg_seen < 3: # print only a few times
print("[SE] gate mean this batch:", gates.mean().item())
self._dbg_seen += 1
out_f = self.classifier_fused(fused)
return out_f, out_img, out_md
# if ablation modes:
if self.mode == "image_only":
return out_img, out_img, None
if self.mode == "metadata_only":
return out_md, None, out_md
class VoteBridge(nn.Module):
def __init__(self, num_classes):
super().__init__()
self.vote_combiner = nn.Linear(num_classes * 2, num_classes) # two sets of logits
def forward(self, out_img, out_md):
votes = torch.cat([out_img, out_md], dim=1)
return self.vote_combiner(votes)
classes/dataset.py
-58
View File
@@ -1,58 +0,0 @@
# dataset.py
from torch.utils.data import Dataset
from PIL import Image
import numpy as np
import torch
class ClinicalDataset(Dataset):
"""Generic dataset wrapping a ClinicalData instance.
Returns (img_tensor, meta_tensor, label)."""
def __init__(
self,
clinical_data,
img_transform,
meta_transform=None,
image_preprocessor=None,
geometry_provider=None,
geometry_dim: int = 0,
):
self.clinical = clinical_data
self.transform_image = img_transform
self.meta_transform = meta_transform or (lambda x: x)
self.image_preprocessor = image_preprocessor
self.geometry_provider = geometry_provider
self.geometry_dim = geometry_dim if geometry_provider is not None else 0
def __len__(self):
return len(self.clinical.df)
def __getitem__(self, idx: int):
row = self.clinical.df.iloc[idx]
# load & transform image
img_path = self.clinical.get_image_path(row)
orig_img = Image.open(img_path).convert("RGB")
img = orig_img
if self.image_preprocessor is not None:
img = self.image_preprocessor(img, img_path)
img_t = self.transform_image(img)
# encode & transform metadata
meta = self.clinical.encode_metadata(row)
meta_t = self.meta_transform(meta)
# label
label = self.clinical.get_label(row)
if self.geometry_dim > 0:
features = None
if self.geometry_provider is not None and hasattr(self.geometry_provider, "geometry_features"):
features = self.geometry_provider.geometry_features(orig_img, img_path)
if features is None:
geom_vec = torch.zeros(self.geometry_dim, dtype=torch.float32)
else:
features = np.asarray(features, dtype=np.float32)
if features.shape[0] != self.geometry_dim:
geom_vec = torch.zeros(self.geometry_dim, dtype=torch.float32)
else:
geom_vec = torch.from_numpy(features)
return img_t, meta_t, geom_vec, label
return img_t, meta_t, label
classes/early_stop.py
-89
View File
@@ -1,89 +0,0 @@
import math, copy, torch
class EarlyStopper:
def __init__(self, monitor: str, mode: str = "auto",
patience: int = 5, min_delta: float = 0.0,
save_path: str | None = None, restore_best: bool = True):
"""
monitor: key in your epoch row, e.g. 'eval_loss', 'auc_fused', 'acc_fused'
mode: 'max' (higher is better), 'min', or 'auto' (min for '*loss*', else max)
patience: epochs without improvement before stopping
min_delta: required improvement magnitude
save_path: optional .pth file to save best weights each time it improves
restore_best: if True, load best weights back at the end
"""
self.monitor = monitor
if mode == "auto":
mode = "min" if "loss" in monitor.lower() else "max"
self.mode = mode
self.patience = int(patience)
self.min_delta = float(min_delta)
self.save_path = save_path
self.restore_best = restore_best
self.best = -math.inf if mode == "max" else math.inf
self.bad_epochs = 0
self.best_state = None
self.best_epoch = -1
self.last_improved = False
def _is_better(self, val):
if val is None or (isinstance(val, float) and math.isnan(val)):
return False
if self.mode == "max":
return val > (self.best + self.min_delta)
else:
return val < (self.best - self.min_delta)
def step(self, metrics: dict, trainer, epoch: int) -> bool:
val = metrics.get(self.monitor, None)
improved = self._is_better(val)
self.last_improved = improved
if improved:
self.best = val
self.best_epoch = epoch
self.bad_epochs = 0
# snapshot + optional save
state = {
"img_tower": trainer.img_tower.state_dict(),
"md_tower": trainer.md_tower.state_dict(),
"optimizer": trainer.optimizer.state_dict(),
}
if hasattr(trainer, "bridge"): state["bridge"] = trainer.bridge.state_dict()
if hasattr(trainer, "head_img"): state["head_img"] = trainer.head_img.state_dict()
if hasattr(trainer, "head_md"): state["head_md"] = trainer.head_md.state_dict()
# keep an in-memory copy for restore(); file save is optional
self.best_state = copy.deepcopy(state)
if self.save_path: torch.save(state, self.save_path)
print(f"[early] ↑ new best {self.monitor}={val:.5f} at epoch {epoch+1}")
else:
self.bad_epochs += 1
stop = self.bad_epochs >= self.patience
if stop:
print(f"[early] stopping: no improvement in {self.patience} epochs "
f"(best {self.monitor}={self.best:.5f} @ epoch {self.best_epoch+1})")
return stop
def restore(self, trainer):
if not self.restore_best:
return
# Prefer in-memory best state; otherwise try loading from save_path
st = self.best_state
if st is None and self.save_path:
try:
st = torch.load(self.save_path, map_location="cpu")
except Exception:
st = None
if st is None:
return
trainer.img_tower.load_state_dict(st["img_tower"])
trainer.md_tower.load_state_dict(st["md_tower"])
if "bridge" in st and hasattr(trainer, "bridge"):
trainer.bridge.load_state_dict(st["bridge"])
if "head_img" in st and hasattr(trainer, "head_img"):
trainer.head_img.load_state_dict(st["head_img"])
if "head_md" in st and hasattr(trainer, "head_md"):
trainer.head_md.load_state_dict(st["head_md"])
trainer.optimizer.load_state_dict(st["optimizer"])
classes/frontend.py
-1402
View File
File diff suppressed because it is too large Load Diff
classes/hypertower.py
-1696
View File
File diff suppressed because it is too large Load Diff
classes/image_tower.py
-125
View File
@@ -1,125 +0,0 @@
# classes/image_tower.py
from __future__ import annotations
import math
from typing import Optional
import torch
from torch import nn
from torchvision import transforms
from classes.backbones import BACKBONES, list_names, load_backbone_weights
from classes.SE_attention import SEBlock
def build_backbone(name: str, freeze_ratio: float = 0.0, augment: bool = True):
"""
Operational builder:
- instantiate with DEFAULT weights
- strip classifier → features
- apply ratio-based freezing over coarse blocks
- return (model, out_dim, transform)
"""
key = (name or "").lower()
if key not in BACKBONES:
raise ValueError(f"Unsupported backbone '{name}'. Valid options: {list_names()}")
spec = BACKBONES[key]
m = spec.ctor(weights=spec.weights_default)
out_dim, m = spec.strip(m)
load_backbone_weights(key, m)
# transforms: use the weights’ mean/std, but keep your augmentation pipeline
mean = getattr(spec.weights_default, "meta", {}).get("mean", (0.485, 0.456, 0.406))
std = getattr(spec.weights_default, "meta", {}).get("std", (0.229, 0.224, 0.225))
crop = 299 if key == "inception_v3" else 224
if augment:
transform = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(crop),
transforms.RandomHorizontalFlip(),
transforms.RandomVerticalFlip(),
transforms.RandomRotation(15),
transforms.ColorJitter(0.1, 0.1, 0.1, 0.05),
transforms.ToTensor(),
transforms.Normalize(mean=mean, std=std),
])
else:
transform = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(crop),
transforms.ToTensor(),
transforms.Normalize(mean=mean, std=std),
])
# ratio-based freezing: freeze earliest floor(N * freeze_ratio) blocks
fr = max(0.0, min(1.0, float(freeze_ratio)))
blocks = spec.blocks(m)
n = len(blocks)
freeze_n = int(math.floor(n * fr))
for b in blocks[:freeze_n]:
for p in b.parameters():
p.requires_grad = False
return m, out_dim, transform
class ImageTower(nn.Module):
"""
Vision backbone → pooled features.
- backbone: one of list_names() (default 'efficientnet_b0')
- always DEFAULT torchvision weights
- freeze_ratio ∈ [0,1] freezes earliest floor(N*freeze_ratio) blocks
- returns [N, out_dim] features from backbone forward
"""
def __init__(self, backbone: str = "efficientnet_b0", freeze_ratio: float = 0.0,
use_se: bool = False, se_reduction: int = 16, se_pre_norm: bool = True,
augment: bool = True, geometry_dim: int = 0):
super().__init__()
self.backbone, base_dim, self.transform = build_backbone(backbone, freeze_ratio, augment=augment)
self._name = backbone
# Keep ordered blocks for dynamic freezing/thawing
key = (self._name or "").lower()
self._spec = BACKBONES[key]
self._blocks = self._spec.blocks(self.backbone)
# Optional tower-level SE over the final feature vector
self.base_dim = base_dim
self.geometry_dim = max(0, int(geometry_dim))
self.out_dim = self.base_dim + self.geometry_dim
self.tower_ln = nn.LayerNorm(self.base_dim) if se_pre_norm else nn.Identity()
self.tower_se = SEBlock(self.base_dim, reduction=se_reduction, residual=True) if use_se else None
def forward(self, x: torch.Tensor, geometry: Optional[torch.Tensor] = None) -> torch.Tensor:
y = self.backbone(x)
# sanity: pooled features, not logits
assert y.dim() == 2 and y.size(1) == self.base_dim, \
f"Expected features [N,{self.base_dim}], got {tuple(y.shape)}"
if self.tower_se is not None:
y, _ = self.tower_se(self.tower_ln(y))
if self.geometry_dim > 0:
if geometry is None or geometry.numel() == 0:
geom = torch.zeros(y.size(0), self.geometry_dim, device=y.device, dtype=y.dtype)
else:
if geometry.dim() == 1:
geom = geometry.unsqueeze(0)
else:
geom = geometry
geom = geom.to(device=y.device, dtype=y.dtype)
if geom.size(0) != y.size(0):
raise ValueError(f"Geometry batch size mismatch: {geom.size(0)} vs {y.size(0)}")
if geom.size(1) != self.geometry_dim:
raise ValueError(f"Expected geometry dim {self.geometry_dim}, got {geom.size(1)}")
y = torch.cat([y, geom], dim=1)
return y
def set_freeze_ratio(self, ratio: float):
"""Dynamically freeze earliest floor(N*ratio) backbone blocks.
ratio in [0,1]."""
r = max(0.0, min(1.0, float(ratio)))
n = len(self._blocks)
freeze_n = int(math.floor(n * r))
# Unfreeze all first
for b in self._blocks:
for p in b.parameters():
p.requires_grad = True
# Freeze earliest blocks
for b in self._blocks[:freeze_n]:
for p in b.parameters():
p.requires_grad = False
classes/md_tower.py
-54
View File
@@ -1,54 +0,0 @@
# md_tower.py
import torch
import torch.nn as nn
from classes import ClinicalData
from classes.SE_attention import SEBlock
class MDTower(nn.Module):
"""MLP over ClinicalData.vectorize_row outputs (convert to torch inside tower)."""
def __init__(self, clinical_data: ClinicalData, hidden_dim: int = 128, dropout: float = 0.1,
use_se: bool = False, se_reduction: int = 16, se_pre_norm: bool = True):
super().__init__()
self.feature_dim = clinical_data.feature_dim
self.out_dim = hidden_dim
# two-block MLP so we can optionally freeze/thaw per block
self.block0 = nn.Sequential(
nn.Linear(self.feature_dim, hidden_dim),
nn.LayerNorm(hidden_dim),
nn.ReLU(inplace=True),
nn.Dropout(dropout),
)
self.block1 = nn.Sequential(
nn.Linear(hidden_dim, hidden_dim),
nn.ReLU(inplace=True),
)
self.net = nn.Sequential(self.block0, self.block1)
self.tower_ln = nn.LayerNorm(hidden_dim) if se_pre_norm else nn.Identity()
self.tower_se = SEBlock(hidden_dim, reduction=se_reduction, residual=True) if use_se else None
def forward(self, meta_np_or_torch) -> torch.Tensor:
if isinstance(meta_np_or_torch, torch.Tensor):
x = meta_np_or_torch
else:
x = torch.as_tensor(meta_np_or_torch, dtype=torch.float32)
h = self.net(x)
if self.tower_se is not None:
h, _ = self.tower_se(self.tower_ln(h))
return h
def set_freeze_ratio(self, ratio: float):
"""Optionally freeze earliest blocks of the MLP.
With two blocks, ratio≥0.5 freezes block0; ratio≥1.0 freezes both."""
r = max(0.0, min(1.0, float(ratio)))
# Unfreeze all
for p in self.block0.parameters():
p.requires_grad = True
for p in self.block1.parameters():
p.requires_grad = True
# Freeze earliest blocks based on ratio threshold
if r >= 0.5:
for p in self.block0.parameters():
p.requires_grad = False
if r >= 1.0:
for p in self.block1.parameters():
p.requires_grad = False
classes/papila_builders.py
-99
View File
@@ -1,99 +0,0 @@
# papila_builders.py
from typing import List, Dict
import numpy as np
import pandas as pd
from classes import ClinicalData # adjust import path if needed
# ---- Pachymetry → IOP correction (per PAPILA Table 3) ----
_PACHY_TABLE: Dict[int, int] = {
475:+5, 485:+4, 495:+4, 505:+3, 515:+2, 525:+1, 535:+1,
545: 0, 555:-1, 565:-1, 575:-2, 585:-3, 595:-4, 605:-4, 615:-5,
}
_PACHY_KEYS = np.array(sorted(_PACHY_TABLE.keys()))
def _nearest_pachy_key(x: float) -> int:
idx = int(np.argmin(np.abs(_PACHY_KEYS - float(x))))
return int(_PACHY_KEYS[idx])
def _pick_iop(row: pd.Series) -> float:
"""Prefer Pneumatic, else Perkins; may return NaN."""
raw = row["Pneumatic"] if not pd.isna(row.get("Pneumatic", np.nan)) else row.get("Perkins", np.nan)
return float(raw) if not pd.isna(raw) else np.nan
def _correct_iop(raw_iop: float, pachy: float) -> float:
"""Return corrected IOP using nearest pachymetry bin; if pachy missing, return raw."""
if pd.isna(raw_iop):
return np.nan
if pd.isna(pachy):
return float(raw_iop)
key = _nearest_pachy_key(float(pachy))
return float(raw_iop) + float(_PACHY_TABLE[key])
def _apply_iop_and_drop_md(df: pd.DataFrame) -> pd.DataFrame:
"""Add IOP_raw/IOP_corr and drop VF_MD if present (in-place safe)."""
# IOP_raw
df["IOP_raw"] = df.apply(_pick_iop, axis=1)
# IOP_corr
pachy = df.get("Pachymetry", pd.Series(np.nan, index=df.index))
df["IOP_corr"] = [
_correct_iop(r, p) for r, p in zip(df["IOP_raw"].values, pachy.values)
]
# Drop VF_MD if present
if "VF_MD" in df.columns:
df.drop(columns=["VF_MD"], inplace=True)
return df
def build_papila_clinical(
image_dir: str,
clinical_dir: str,
label_col: str,
cat_cols: List[str],
n_splits: int = 5,
random_seed: int = 42,
) -> ClinicalData:
"""
Build ClinicalData exactly like the user's original build_clinical:
- add_df(OD), set eyeID='OD'
- add_df(OS), set eyeID='OS'
- normalize 'Patient ID' on frames
THEN:
- compute IOP_raw / IOP_corr on each frame
- drop VF_MD
- refresh master df + kfold indices
"""
clinical = ClinicalData(
image_dir=image_dir,
clinical_dir=clinical_dir,
label_col=label_col,
cat_cols=cat_cols,
n_splits=n_splits,
random_seed=random_seed,
)
# --- Load exactly like original build_clinical ---
clinical.add_df(pd.read_excel(f"{clinical_dir}/patient_data_od.xlsx", header=1), id_column="ID")
clinical.frames[0]["eyeID"] = "OD"
clinical.add_df(pd.read_excel(f"{clinical_dir}/patient_data_os.xlsx", header=1), id_column="ID")
clinical.frames[1]["eyeID"] = "OS"
# Normalize 'Patient ID' on the per-eye frames (string → int)
for frame in clinical.frames:
frame["Patient ID"] = frame["Patient ID"].astype(str).str.extract(r"(\d+)")[0].astype(int)
# Build initial master as in original
clinical._refresh_master_df()
# --- Post-processing ON THE FRAMES (so everything stays consistent) ---
for i in range(len(clinical.frames)):
clinical.frames[i] = _apply_iop_and_drop_md(clinical.frames[i])
# Refresh master again so IOP_raw/IOP_corr & MD removal propagate
clinical._refresh_master_df()
clinical._build_kfold_indices()
return clinical
classes/tower_watcher.py
-149
View File
@@ -1,149 +0,0 @@
# tower_watcher.py
import matplotlib.pyplot as plt
# tower_watcher.py
import matplotlib.pyplot as plt
class TowerWatcher:
"""
Live monitor:
- Cumulative batch-level: loss & accuracy per batch across all epochs.
- Epoch batch-level: loss & accuracy per batch within the current epoch (resets each epoch).
- TP/FP/TN/FN bar charts per tower, one chart each, new group each epoch.
"""
def __init__(self):
plt.ion()
# 2 line plots (cum loss, cum acc), 2 line plots (epoch loss, epoch acc), 3 bar plots
self.fig, self.axs = plt.subplots(7, 1, figsize=(10, 28))
self.reset()
def reset(self):
# Cumulative batch-level
self.global_batches = []
self.loss_cum = {"fusion": [], "image": [], "meta": []}
self.acc_cum = {"fusion": [], "image": [], "meta": []}
# Epoch batch-level
self.epoch_batches = []
self.loss_epoch_batch = {"fusion": [], "image": [], "meta": []}
self.acc_epoch_batch = {"fusion": [], "image": [], "meta": []}
# Epoch markers for cum plots
self.epoch_markers = []
# Stats per epoch for bars
self.epoch_stats = {"fusion": [], "image": [], "meta": []}
# Track current epoch
self.current_epoch = -1
def on_epoch_start(self, epoch):
# mark epoch boundary in cumulative
x = self.global_batches[-1] + 1 if self.global_batches else 0
self.epoch_markers.append(x)
# reset epoch batch-level data
self.epoch_batches = []
for d in [self.loss_epoch_batch, self.acc_epoch_batch]:
for k in d:
d[k].clear()
self.current_epoch = epoch
def on_batch_end(self, idx, stats: dict):
# Cumulative
self.global_batches.append(len(self.global_batches) + 1)
for key, lk, ak in [
("fusion", "loss_f", "acc_f"),
("image", "loss_i", "acc_i"),
("meta", "loss_m", "acc_m"),
]:
self.loss_cum[key].append(stats.get(lk, 0))
self.acc_cum[key].append(stats.get(ak, 0))
# Epoch-level
self.epoch_batches.append(len(self.epoch_batches) + 1)
for key, lk, ak in [
("fusion", "loss_f", "acc_f"),
("image", "loss_i", "acc_i"),
("meta", "loss_m", "acc_m"),
]:
self.loss_epoch_batch[key].append(stats.get(lk, 0))
self.acc_epoch_batch[key].append(stats.get(ak, 0))
# redraw
self._draw_batch_plots()
def on_epoch_end(self, epoch, stats: dict):
# record per-epoch TP/FP/TN/FN
for key in ["fusion", "image", "meta"]:
self.epoch_stats[key].append(
{
"tp": stats.get("tp", 0),
"fp": stats.get("fp", 0),
"tn": stats.get("tn", 0),
"fn": stats.get("fn", 0),
}
)
self._draw_epoch_bars()
def _draw_batch_plots(self):
# Cumulative Loss
ax = self.axs[0]
ax.clear()
ax.plot(self.global_batches, self.loss_cum["fusion"], label="Fusion")
ax.plot(self.global_batches, self.loss_cum["image"], label="Image Tower")
ax.plot(self.global_batches, self.loss_cum["meta"], label="MD Tower")
for x in self.epoch_markers:
ax.axvline(x=x, color="gray", linestyle="--")
ax.set_ylabel("Cumulative Loss")
ax.legend()
# Epoch Loss
ax = self.axs[1]
ax.clear()
ax.plot(self.epoch_batches, self.loss_epoch_batch["fusion"], label="Fusion")
ax.plot(self.epoch_batches, self.loss_epoch_batch["image"], label="Image Tower")
ax.plot(self.epoch_batches, self.loss_epoch_batch["meta"], label="MD Tower")
ax.set_ylabel(f"Epoch {self.current_epoch+1} Loss")
ax.set_xlabel("Batch (Epoch)")
ax.legend()
# Cumulative Accuracy
ax = self.axs[2]
ax.clear()
ax.plot(self.global_batches, self.acc_cum["fusion"], label="Fusion")
ax.plot(self.global_batches, self.acc_cum["image"], label="Image Tower")
ax.plot(self.global_batches, self.acc_cum["meta"], label="MD Tower")
for x in self.epoch_markers:
ax.axvline(x=x, color="gray", linestyle="--")
ax.set_ylabel("Cumulative Accuracy")
ax.legend()
# Epoch Accuracy
ax = self.axs[3]
ax.clear()
ax.plot(self.epoch_batches, self.acc_epoch_batch["fusion"], label="Fusion")
ax.plot(self.epoch_batches, self.acc_epoch_batch["image"], label="Image Tower")
ax.plot(self.epoch_batches, self.acc_epoch_batch["meta"], label="MD Tower")
ax.set_ylabel(f"Epoch {self.current_epoch+1} Accuracy")
ax.set_xlabel("Batch (Epoch)")
ax.legend()
plt.pause(0.01)
def _draw_epoch_bars(self):
# Bar charts per tower
for i, key in enumerate(["fusion", "image", "meta"]):
ax = self.axs[4 + i]
ax.clear()
data = self.epoch_stats[key]
epochs = list(range(1, len(data) + 1))
tp = [d["tp"] for d in data]
fp = [d["fp"] for d in data]
tn = [d["tn"] for d in data]
fn = [d["fn"] for d in data]
width = 0.2
ax.bar([e - width for e in epochs], tp, width, label="TP")
ax.bar(epochs, fp, width, label="FP")
ax.bar([e + width for e in epochs], tn, width, label="TN")
ax.bar([e + 2 * width for e in epochs], fn, width, label="FN")
ax.set_title(f"{key.title()} Tower Stats")
ax.set_xlabel("Epoch")
ax.set_ylabel("Count")
ax.legend()
plt.pause(0.01)
classes/SE_attention.py → classes/v2/SE_attention.py
View File
classes/backbones.py → classes/v2/backbones.py
+1 -1
View File
@@ -15,7 +15,7 @@ class BackboneSpec:
strip: Callable[[nn.Module], tuple] # fn(model)->(out_dim, model_no_head)
blocks: Callable[[nn.Module], List[nn.Module]] # fn(model)->ordered blocks for freezing
REFUGELIKE_BACKBONE_PATH = Path("models/refuge/classifier/refugelike_backbone.pt")
REFUGELIKE_BACKBONE_PATH = Path("models/v2/refuge/refugelike_backbone.pt")
REFUGE_DENSENET_PATH = Path("models/refuge/classifier/refuge_densenet_backbone.pt")
REFUGE_EFFICIENT_B0_PATH = Path("models/refuge/classifier/refuge_efficient_b0_backbone.pt")
REFUGE_EFFICIENT_B7_PATH = Path("models/refuge/classifier/refuge_efficient_b7_backbone.pt")
classes/v2/bridges.py
+1 -1
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
import torch
import torch.nn as nn
from classes.SE_attention import SEBlock, SEGateLogger
from classes.v2.SE_attention import SEBlock, SEGateLogger
class Bridge(nn.Module):
classes/v2/croppers.py
+27 -4
View File
@@ -10,9 +10,30 @@ import torch
from PIL import Image, ImageDraw
from torchvision import transforms
from classes.geometry_features import compute_geometry_features, disc_cup_from_mask_image
from classes.refuge_classification import _geometry_from_mask
from classes.unet_segmenter import UNetSegmenter
from classes.v2.geometry_features import compute_geometry_features, disc_cup_from_mask_image
from classes.v2.unet_segmenter import UNetSegmenter
def _geometry_from_mask(mask: np.ndarray, scale: float) -> Dict:
mask = np.asarray(mask) > 0
coords = np.argwhere(mask)
if coords.size == 0:
raise RuntimeError("Empty mask; cannot derive geometry")
ys, xs = coords[:, 0], coords[:, 1]
centre_x = float(xs.mean())
centre_y = float(ys.mean())
width = float(xs.max() - xs.min())
height = float(ys.max() - ys.min())
diameter = max(width, height)
radius = diameter / 2.0
crop_radius = radius * scale
return {
"centre_x": centre_x,
"centre_y": centre_y,
"radius": radius,
"crop_radius": crop_radius,
"crop_size": crop_radius * 2.0,
}
class UNetImageCropper:
@@ -61,7 +82,9 @@ class UNetImageCropper:
def _infer_masks(self, image: Image.Image) -> Optional[Tuple[np.ndarray, np.ndarray]]:
resized = self.segmenter.preprocess_image(image)
tensor = self.to_tensor(resized).unsqueeze(0).to(self.segmenter.device)
tensor = self.segmenter._normalize_tensor(
self.to_tensor(resized).to(self.segmenter.device)
).unsqueeze(0)
with torch.no_grad():
logits = self.segmenter.model(tensor)
classes/v2/dataset.py
+9 -1
View File
@@ -16,6 +16,7 @@ class ClinicalDataset(Dataset):
image_preprocessor=None,
geometry_provider=None,
geometry_dim: int = 0,
image_cache: "dict | None" = None,
):
self.clinical = clinical_data
self.transform_image = img_transform
@@ -23,6 +24,7 @@ class ClinicalDataset(Dataset):
self.image_preprocessor = image_preprocessor
self.geometry_provider = geometry_provider
self.geometry_dim = geometry_dim if geometry_provider is not None else 0
self.image_cache = image_cache
def __len__(self):
return len(self.clinical.df)
@@ -31,7 +33,13 @@ class ClinicalDataset(Dataset):
row = self.clinical.df.iloc[idx]
# load & transform image
img_path = self.clinical.get_image_path(row)
orig_img = Image.open(img_path).convert("RGB")
cache_key = str(img_path)
if self.image_cache is not None and cache_key in self.image_cache:
orig_img = Image.fromarray(self.image_cache[cache_key])
else:
orig_img = Image.open(img_path).convert("RGB")
if self.image_cache is not None:
self.image_cache[cache_key] = np.asarray(orig_img, dtype=np.uint8)
img = orig_img
if self.image_preprocessor is not None:
img = self.image_preprocessor(img, img_path)
classes/geometry_features.py → classes/v2/geometry_features.py
View File
classes/v2/loader_factory.py
+2
View File
@@ -206,6 +206,7 @@ def make_loader(
*,
image_transform,
image_preprocessor=None,
image_cache=None,
batch_size: int,
shuffle: bool,
num_workers: int,
@@ -216,6 +217,7 @@ def make_loader(
slots,
image_transform=image_transform,
image_preprocessor=image_preprocessor,
image_cache=image_cache,
)
return DataLoader(
ds,
classes/v2/metrics.py
+22 -10
View File
@@ -12,6 +12,7 @@ from sklearn.metrics import (
matthews_corrcoef,
recall_score,
roc_auc_score,
roc_curve,
)
@@ -142,26 +143,37 @@ def compute_extended_metrics(
# ---------------------------------------------------------------------------
def tune_binary_threshold(y_true: np.ndarray, p1: np.ndarray) -> float:
if y_true.size == 0:
"""Pick threshold via Youden's J (sensitivity + specificity − 1).
This is class-distribution independent, unlike maximising raw accuracy,
which is biased toward the majority class on imbalanced validation sets.
Falls back to 0.5 if both classes are not present.
"""
if y_true.size == 0 or len(np.unique(y_true)) < 2:
return 0.5
grid = np.linspace(0.0, 1.0, 1001)
best_t, best_acc = 0.5, -1.0
for t in grid:
pred = (p1 >= t).astype(int)
acc = float((pred == y_true).mean())
if acc > best_acc or (acc == best_acc and abs(t - 0.5) < abs(best_t - 0.5)):
best_acc, best_t = acc, float(t)
return best_t
fpr, tpr, thresholds = roc_curve(y_true, p1)
j = tpr + (1.0 - fpr) - 1.0
return float(thresholds[np.argmax(j)])
def multiclass_acc_with_bias(y_true: np.ndarray, probs: np.ndarray, bias: np.ndarray) -> float:
"""Balanced accuracy (mean per-class recall) after applying log-space bias."""
if y_true.size == 0:
return float("nan")
logits = np.log(np.clip(probs, 1e-8, 1.0)) + bias.reshape(1, -1)
return float((np.argmax(logits, axis=1) == y_true).mean())
preds = np.argmax(logits, axis=1)
classes = np.unique(y_true)
per_class = [(preds[y_true == c] == c).mean() for c in classes]
return float(np.mean(per_class))
def tune_multiclass_bias(y_true: np.ndarray, probs: np.ndarray, *, iters: int = 2) -> np.ndarray:
"""Grid-search per-class log-space bias to maximise balanced accuracy.
Balanced accuracy (mean per-class recall) is class-distribution independent,
unlike raw accuracy which is biased toward the majority class on imbalanced
validation sets.
"""
if y_true.size == 0 or probs.size == 0:
return np.zeros((0,), dtype=float)
c = probs.shape[1]
classes/v2/models.py
+37 -1
View File
@@ -219,6 +219,15 @@ def _set_single_phase(model: SingleEyeHT, phase: str) -> None:
# Ablation modes have no fusion bridge; fused_warmup is meaningless — treat as tower_warmup
if bridge_mode in ("image_only", "metadata_only") and phase == "fused_warmup":
phase = "tower_warmup"
if phase == "md_warmup":
_set_requires_grad(model.img_tower, False)
_set_requires_grad(model.md_tower, True)
_set_requires_grad(model.bridge.classifier_img, False)
_set_requires_grad(model.bridge.classifier_md, True)
_set_requires_grad(model.bridge.W_img, False)
_set_requires_grad(model.bridge.W_md, False)
_set_requires_grad(model.bridge.classifier_fused, False)
return
if phase == "tower_warmup":
_set_requires_grad(model.img_tower, bridge_mode != "metadata_only")
_set_requires_grad(model.md_tower, bridge_mode != "image_only")
@@ -274,6 +283,7 @@ def train_single_epoch(
*,
phase: str,
bcd_prob: float = 0.5,
tower_loss_mode: str = "bcd",
) -> tuple[float, float]:
model.train()
_set_single_phase(model, phase)
@@ -282,12 +292,27 @@ def train_single_epoch(
x = batch.get("image_1")
m = batch.get("matrix_1")
y = batch.get("label_1")
if phase == "md_warmup":
if not torch.is_tensor(m):
continue
m = m.to(device)
y = _to_label_tensor(y, device)
md_feats = model.md_tower(m)
logits = model.bridge.classifier_md(md_feats)
loss = F.cross_entropy(logits, y)
opt.zero_grad(); loss.backward(); opt.step()
bs = y.shape[0]
total_loss += float(loss.item()) * bs
total_correct += int((logits.argmax(1) == y).sum())
total_n += bs
continue
if not torch.is_tensor(x) or not torch.is_tensor(m):
continue
x = x.to(device)
m = m.to(device)
y = _to_label_tensor(y, device)
bridge_mode = model.bridge.mode
img_feats = None if bridge_mode == "metadata_only" else model.img_tower(x)
md_feats = None if bridge_mode == "image_only" else model.md_tower(m)
@@ -313,6 +338,11 @@ def train_single_epoch(
elif bridge_mode == "image_only":
logits = model.bridge.classifier_img(img_feats)
loss = F.cross_entropy(logits, y)
elif tower_loss_mode == "all":
loss_i = F.cross_entropy(model.bridge.classifier_img(img_feats), y)
loss_m = F.cross_entropy(model.bridge.classifier_md(md_feats), y)
logits, _, _ = model.bridge(img_feats, md_feats)
loss = F.cross_entropy(logits, y) + loss_i + loss_m
elif random() < bcd_prob:
if random() < 0.5:
logits = model.bridge.classifier_img(img_feats)
@@ -344,6 +374,7 @@ def train_bilateral_epoch(
*,
phase: str,
bcd_prob: float = 0.5,
tower_loss_mode: str = "bcd",
) -> tuple[float, float]:
model.train()
_set_bilateral_phase(model, phase)
@@ -370,7 +401,12 @@ def train_bilateral_epoch(
logits, _, _ = model.bridge(joint_img, joint_md)
loss = F.cross_entropy(logits, y)
else:
if random() < bcd_prob:
if tower_loss_mode == "all":
loss_i = F.cross_entropy(model.aux_img(joint_img), y)
loss_m = F.cross_entropy(model.aux_md(joint_md), y)
logits, _, _ = model.bridge(joint_img, joint_md)
loss = F.cross_entropy(logits, y) + loss_i + loss_m
elif random() < bcd_prob:
if random() < 0.5:
logits = model.aux_img(joint_img)
else:
classes/v2/papila_builders.py
+100 -13
View File
@@ -1,6 +1,6 @@
from __future__ import annotations
from typing import Dict, List
from typing import Callable, Dict, List, Optional
import numpy as np
import pandas as pd
@@ -33,10 +33,80 @@ def _nearest_pachy_key(x: float) -> int:
return int(_PACHY_KEYS[idx])
def _pick_iop(row: pd.Series) -> float:
"""Prefer Pneumatic, else Perkins; may return NaN."""
raw = row["Pneumatic"] if not pd.isna(row.get("Pneumatic", np.nan)) else row.get("Perkins", np.nan)
return float(raw) if not pd.isna(raw) else np.nan
def _fit_perkins_converter(
frames: List[pd.DataFrame], method: str
) -> Callable[[float, Optional[float]], float]:
"""
Fit a Perkins→Pneumatic converter from pooled paired observations across all frames.
Returns a callable: converter(perkins_value, pachymetry_value) -> float.
Supported methods: "ratio", "ols", "lad", "multi".
"""
combined = pd.concat(frames, ignore_index=True)
paired = combined.dropna(subset=["Pneumatic", "Perkins"])
pneumatic = paired["Pneumatic"].values.astype(float)
perkins = paired["Perkins"].values.astype(float)
if len(paired) == 0:
raise ValueError("No paired Pneumatic+Perkins observations found; cannot fit converter.")
if method == "ratio":
ratio = float((pneumatic / perkins).mean())
def converter_ratio(p: float, pachy: Optional[float] = None) -> float:
return p * ratio
return converter_ratio
elif method == "ols":
from scipy import stats as _stats
slope, intercept, *_ = _stats.linregress(perkins, pneumatic)
slope, intercept = float(slope), float(intercept)
def converter_ols(p: float, pachy: Optional[float] = None) -> float:
return p * slope + intercept
return converter_ols
elif method == "lad":
from scipy import stats as _stats
from scipy.optimize import minimize as _minimize
slope0, intercept0, *_ = _stats.linregress(perkins, pneumatic)
def _lad_loss(params):
a, b = params
return np.abs(pneumatic - (a * perkins + b)).mean()
res = _minimize(_lad_loss, x0=[slope0, intercept0], method="Nelder-Mead")
slope, intercept = float(res.x[0]), float(res.x[1])
def converter_lad(p: float, pachy: Optional[float] = None) -> float:
return p * slope + intercept
return converter_lad
elif method == "multi":
from numpy.linalg import lstsq as _lstsq
paired_multi = combined.dropna(subset=["Pneumatic", "Perkins", "Pachymetry"])
if len(paired_multi) == 0:
raise ValueError("No paired Pneumatic+Perkins+Pachymetry rows; cannot fit multi method.")
pneu = paired_multi["Pneumatic"].values.astype(float)
perk = paired_multi["Perkins"].values.astype(float)
pachy_vals = paired_multi["Pachymetry"].values.astype(float)
X = np.column_stack([perk, pachy_vals, np.ones(len(perk))])
coeffs, *_ = _lstsq(X, pneu, rcond=None)
slope, pachy_coef, intercept = float(coeffs[0]), float(coeffs[1]), float(coeffs[2])
pachy_fallback = float(pachy_vals.mean())
def converter_multi(p: float, pachy: Optional[float] = None) -> float:
pv = pachy if (pachy is not None and not np.isnan(pachy)) else pachy_fallback
return p * slope + pachy_coef * pv + intercept
return converter_multi
else:
raise ValueError(f"Unknown iop_corr_method: {method!r}. Choose ratio/ols/lad/multi.")
def _pick_iop(row: pd.Series, converter: Callable) -> float:
"""Prefer Pneumatic; convert Perkins to Pneumatic scale if Pneumatic is absent."""
pneumatic = row.get("Pneumatic", np.nan)
if not pd.isna(pneumatic):
return float(pneumatic)
perkins = row.get("Perkins", np.nan)
if pd.isna(perkins):
return np.nan
pachy = row.get("Pachymetry", np.nan)
return converter(float(perkins), None if pd.isna(pachy) else float(pachy))
def _correct_iop(raw_iop: float, pachy: float) -> float:
@@ -49,14 +119,20 @@ def _correct_iop(raw_iop: float, pachy: float) -> float:
return float(raw_iop) + float(_PACHY_TABLE[key])
def _apply_iop_and_drop_md(df: pd.DataFrame) -> pd.DataFrame:
def _apply_iop_and_drop_md(
df: pd.DataFrame,
converter: Callable,
drop_raw: bool = False,
) -> pd.DataFrame:
"""Add IOP_raw/IOP_corr and drop source IOP columns + VF_MD if present (in-place safe)."""
df["IOP_raw"] = df.apply(_pick_iop, axis=1)
df["IOP_raw"] = df.apply(lambda row: _pick_iop(row, converter), axis=1)
pachy = df.get("Pachymetry", pd.Series(np.nan, index=df.index))
df["IOP_corr"] = [
_correct_iop(r, p) for r, p in zip(df["IOP_raw"].values, pachy.values)
]
drop_cols = [c for c in ("Pneumatic", "Perkins", "VF_MD") if c in df.columns]
if drop_raw:
drop_cols.append("IOP_raw")
if drop_cols:
df.drop(columns=drop_cols, inplace=True)
return df
@@ -106,6 +182,9 @@ def build_papila_data(
cat_cols: List[str],
n_splits: int = 5,
random_seed: int = 42,
iop_corr_method: str = "ratio",
iop_drop_raw: bool = False,
exclude_cols: Optional[List[str]] = None,
) -> DataBundle:
"""
Build a DataBundle for PAPILA with dataset-specific preprocessing:
@@ -115,12 +194,17 @@ def build_papila_data(
- compute IOP_raw / IOP_corr, drop VF_MD
- build feature typing & folds
"""
_exclude = list(exclude_cols) if exclude_cols else []
# Remove excluded cols from cat_cols too so the bundle doesn't try to encode them
effective_cat_cols = [c for c in cat_cols if c not in _exclude]
bundle = DataBundle(
image_dir=image_dir,
clinical_dir=clinical_dir,
label_col=label_col,
patient_col="Patient ID",
cat_cols=cat_cols,
cat_cols=effective_cat_cols,
n_splits=n_splits,
random_seed=random_seed,
filename_template="RET{pid:03d}{eye}.jpg",
@@ -137,14 +221,17 @@ def build_papila_data(
frame["Patient ID"] = frame["Patient ID"].astype(str).str.extract(r"(\d+)")[0].astype(int)
_canonicalize_eye_column(frame)
bundle.add_df(od, id_column="ID")
bundle.add_df(os, id_column="ID")
bundle.add_df(od, id_column="ID", exclude_cols=_exclude or None)
bundle.add_df(os, id_column="ID", exclude_cols=_exclude or None)
converter = _fit_perkins_converter(bundle.frames, method=iop_corr_method)
for i in range(len(bundle.frames)):
bundle.frames[i] = _apply_iop_and_drop_md(bundle.frames[i])
bundle.frames[i] = _apply_iop_and_drop_md(
bundle.frames[i], converter=converter, drop_raw=iop_drop_raw
)
bundle._refresh_master_df()
bundle._infer_or_validate_feature_types()
bundle._refresh_master_df(exclude_cols=_exclude or None)
bundle._infer_or_validate_feature_types(exclude_cols=_exclude or None)
bundle._compute_numeric_stats()
bundle._build_cat_maps()
bundle._compute_feature_dim()
classes/v2/predictions.py
+194
View File
@@ -0,0 +1,194 @@
"""PredictionStore — unified per-epoch prediction tensor across all folds.
Tensor shape: (n_folds, n_epochs, n_samples, n_heads, n_classes)
The meaning of "sample" depends on tower_mode:
single — each eye is a sample; sample_ids like "5OD", "14OS"
ensemble — each patient is a sample; sample_ids like "5", "14"
fused — same as ensemble
bilateral— same as ensemble
Head names by mode:
single : ["fused", "img", "md"]
ensemble : ["od_fused", "od_img", "od_md", "os_fused", "os_img", "os_md"]
fused : ["od_fused", "od_img", "od_md", "os_fused", "os_img", "os_md", "bilat_fused"]
bilateral : ["fused", "img_joint", "md_joint"]
"""
from __future__ import annotations
from pathlib import Path
from typing import Sequence
import numpy as np
def head_names_for_mode(tower_mode: str, *, fused_head: bool = False) -> list[str]:
"""Return canonical head name list for a given tower_mode."""
if tower_mode in ("single", "classic"):
return ["fused", "img", "md"]
if tower_mode == "ensemble":
names = ["od_fused", "od_img", "od_md", "os_fused", "os_img", "os_md"]
return names + ["bilat_fused"] if fused_head else names
if tower_mode == "bilateral":
return ["fused", "img_joint", "md_joint"]
raise ValueError(f"Unknown tower_mode: {tower_mode!r}")
class PredictionStore:
"""
Stores per-epoch predictions for every sample, head, and fold in one tensor.
Usage
-----
# Build once before the fold loop:
store = PredictionStore(
sample_ids=all_eye_or_patient_ids,
y_true=all_labels,
head_names=head_names_for_mode(tower_mode, fused_head=args.fused_head),
n_folds=n_folds,
n_epochs=total_epochs,
n_classes=num_classes,
)
# Inside each epoch, after collecting probs:
store.record(fold, epoch, patient_ids_batch, "od_fused", probs_od)
store.set_split(fold, train_ids, "train")
store.set_split(fold, val_ids, "val")
# After all folds:
store.save(run_dir / "predictions.npz")
# Load and query:
store = PredictionStore.load("predictions.npz")
store.query("5", "od_fused", fold=0) # → (n_epochs, n_classes)
store.query("5", "od_fused") # → (n_folds, n_epochs, n_classes)
store.get_split("5", fold=0) # → "train"
"""
def __init__(
self,
sample_ids: Sequence[str],
y_true: Sequence[int],
head_names: Sequence[str],
n_folds: int,
n_epochs: int,
n_classes: int,
):
self.sample_ids = np.array(sample_ids, dtype=object)
self.y_true = np.array(y_true, dtype=np.int64)
self.head_names = np.array(head_names, dtype=object)
self.n_folds = n_folds
self.n_epochs = n_epochs
self.n_classes = n_classes
n_samples = len(self.sample_ids)
n_heads = len(self.head_names)
self.probs = np.full(
(n_folds, n_epochs, n_samples, n_heads, n_classes),
fill_value=np.nan,
dtype=np.float32,
)
self.split = np.full((n_folds, n_samples), fill_value="", dtype=object)
self._sid_index: dict[str, int] = {str(s): i for i, s in enumerate(self.sample_ids)}
self._head_index: dict[str, int] = {str(h): i for i, h in enumerate(self.head_names)}
# ------------------------------------------------------------------
# Writing
# ------------------------------------------------------------------
def record(
self,
fold: int,
epoch: int,
sample_ids: Sequence[str],
head_name: str,
probs: np.ndarray,
) -> None:
"""Record a batch of predictions for one head.
Args:
fold: 0-indexed fold number
epoch: 0-indexed epoch number
sample_ids: sequence of sample ID strings (length B)
head_name: which head — must be in self.head_names
probs: (B, n_classes) probability array
"""
head_idx = self._head_index.get(head_name)
if head_idx is None:
return # head not active in this mode — skip silently
for i, sid in enumerate(sample_ids):
s_idx = self._sid_index.get(str(sid))
if s_idx is not None:
self.probs[fold, epoch, s_idx, head_idx, :] = probs[i]
def set_split(
self,
fold: int,
sample_ids: Sequence[str],
label: str,
) -> None:
"""Label a group of samples as 'train', 'val', or 'holdout' for a fold."""
for sid in sample_ids:
s_idx = self._sid_index.get(str(sid))
if s_idx is not None:
self.split[fold, s_idx] = label
# ------------------------------------------------------------------
# Querying
# ------------------------------------------------------------------
def query(
self,
sample_id: str,
head_name: str,
fold: int | None = None,
) -> np.ndarray:
"""Return epoch-level predictions for one sample + head.
Returns:
fold=None → (n_folds, n_epochs, n_classes)
fold=int → (n_epochs, n_classes)
"""
s_idx = self._sid_index[str(sample_id)]
head_idx = self._head_index[str(head_name)]
if fold is None:
return self.probs[:, :, s_idx, head_idx, :]
return self.probs[fold, :, s_idx, head_idx, :]
def get_split(self, sample_id: str, fold: int) -> str:
"""Return the split label ('train'/'val'/'holdout') for a sample in a fold."""
s_idx = self._sid_index[str(sample_id)]
return str(self.split[fold, s_idx])
# ------------------------------------------------------------------
# Persistence
# ------------------------------------------------------------------
def save(self, path: str | Path) -> None:
np.savez_compressed(
path,
probs=self.probs,
split=self.split,
sample_ids=self.sample_ids,
y_true=self.y_true,
head_names=self.head_names,
)
@classmethod
def load(cls, path: str | Path) -> "PredictionStore":
data = np.load(path, allow_pickle=True)
probs = data["probs"]
n_folds, n_epochs, _, _, n_classes = probs.shape
store = cls(
sample_ids=data["sample_ids"].tolist(),
y_true=data["y_true"],
head_names=data["head_names"].tolist(),
n_folds=n_folds,
n_epochs=n_epochs,
n_classes=n_classes,
)
store.probs = probs
store.split = data["split"]
return store
classes/v2/slot_dataset.py
+61
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Any, Callable, Optional
from pathlib import Path
@@ -45,12 +46,14 @@ class SlotDataset(Dataset):
image_transform: Optional[Callable[[Image.Image], torch.Tensor]] = None,
matrix_transform: Optional[Callable[[Any], torch.Tensor]] = None,
image_preprocessor: Optional[Callable[..., Image.Image]] = None,
image_cache: Optional[dict[str, np.ndarray]] = None,
) -> None:
self.samples = samples
self.slot_descriptors = slot_descriptors
self.image_transform = image_transform or transforms.ToTensor()
self.matrix_transform = matrix_transform or self._default_matrix_transform
self.image_preprocessor = image_preprocessor
self.image_cache = image_cache
def __len__(self) -> int:
return len(self.samples)
@@ -74,14 +77,72 @@ class SlotDataset(Dataset):
raise ValueError("Missing required image slot")
return None
path = Path(value)
cache_key = str(value)
if self.image_cache is not None:
cached = self.image_cache.get(cache_key)
if cached is not None:
return self.image_transform(Image.fromarray(cached, mode="RGB"))
img = Image.open(path).convert("RGB")
if self.image_preprocessor is not None:
try:
img = self.image_preprocessor(img, path)
except TypeError:
img = self.image_preprocessor(img)
if self.image_cache is not None:
self.image_cache[cache_key] = np.asarray(img, dtype=np.uint8)
return self.image_transform(img)
def prebuild_image_cache(self, cache_workers: int = 0) -> None:
"""Pre-populate image_cache for all samples in this dataset."""
if self.image_cache is None:
return
paths = list({
str(record[key])
for record in self.samples
for key, desc in self.slot_descriptors.items()
if desc.kind == "image" and record.get(key) is not None
})
to_warm = [p for p in paths if p not in self.image_cache]
if not to_warm:
return
print(
f"[image_cache] warming {len(to_warm)} images "
f"({len(paths) - len(to_warm)} already cached)",
flush=True,
)
def _warm_one(path_str: str) -> None:
if path_str in self.image_cache:
return
p = Path(path_str)
img = Image.open(p).convert("RGB")
if self.image_preprocessor is not None:
try:
img = self.image_preprocessor(img, p)
except TypeError:
img = self.image_preprocessor(img)
self.image_cache[path_str] = np.asarray(img, dtype=np.uint8)
try:
from tqdm import tqdm
except ImportError:
tqdm = None
if cache_workers <= 1:
it = tqdm(to_warm, desc="Warm image cache", unit="img") if tqdm else to_warm
for path_str in it:
_warm_one(path_str)
else:
with ThreadPoolExecutor(max_workers=cache_workers) as ex:
futures = {ex.submit(_warm_one, p): p for p in to_warm}
it = tqdm(as_completed(futures), total=len(futures), desc="Warm image cache", unit="img") if tqdm else as_completed(futures)
for fut in it:
fut.result()
def _load_matrix(self, value: Any, *, required: bool) -> Optional[torch.Tensor]:
if value is None:
if required:
classes/v2/towers.py
+2 -2
View File
@@ -7,8 +7,8 @@ import torch
from torch import nn
from torchvision import transforms
from classes.backbones import BACKBONES, list_names, load_backbone_weights
from classes.SE_attention import SEBlock
from classes.v2.backbones import BACKBONES, list_names, load_backbone_weights
from classes.v2.SE_attention import SEBlock
from classes.v2.data_bundle import DataBundle
classes/v2/transforms.py
+1 -1
View File
@@ -7,7 +7,7 @@ from PIL import Image
from torchvision import transforms
from classes.backbones import BACKBONES
from classes.v2.backbones import BACKBONES
IMAGENET_MEAN: Tuple[float, float, float] = (0.485, 0.456, 0.406)
classes/unet_segmenter.py → classes/v2/unet_segmenter.py
View File
classes/v2/v2_hypertower.py
+241 -22
View File
@@ -48,6 +48,7 @@ from classes.v2.models import (
train_single_epoch,
)
from classes.v2.papila_builders import build_papila_data
from classes.v2.predictions import PredictionStore, head_names_for_mode
from classes.v2.profiles import build_papila_profile
from classes.v2.results import FoldArtifacts, FoldResult, _f, _nan, _sv
from classes.v2.split_manager import PatientFirstSplitManager
@@ -176,6 +177,8 @@ class V2HyperTower:
ap.add_argument("--clinical-dir", default="Papila/ClinicalData")
ap.add_argument("--label-col", default="Diagnosis")
ap.add_argument("--cat-cols", nargs="*", default=["Gender", "Phakic/Pseudophakic"])
ap.add_argument("--exclude-cols", nargs="*", default=[],
help="Feature columns to exclude entirely from the clinical feature matrix.")
ap.add_argument("--eval-mode", choices=["binary", "multiclass"], default="multiclass")
ap.add_argument(
"--tower-mode", choices=["single", "ensemble", "bilateral", "classic"],
@@ -205,6 +208,9 @@ class V2HyperTower:
help="Single-eye model tower warmup (overrides --warmup-tower-epochs).")
ap.add_argument("--single-warmup-fused-epochs", type=int, default=None,
help="Single-eye model fused warmup (overrides --warmup-fused-epochs).")
ap.add_argument("--warmup-md-epochs", type=int, default=0,
help="MD-only warmup epochs before tower warmup. Trains only md_tower + "
"classifier_md (no CNN forward pass, so 50-100 epochs is cheap).")
ap.add_argument("--bilat-warmup-tower-epochs", type=int, default=None,
help="Bilateral model tower warmup (overrides --warmup-tower-epochs).")
ap.add_argument("--bilat-warmup-fused-epochs", type=int, default=None,
@@ -213,12 +219,22 @@ class V2HyperTower:
ap.add_argument("--lr", type=float, default=1e-4)
ap.add_argument("--bcd-prob", type=float, default=0.5,
help="Tower-only step probability during main phase (per model).")
ap.add_argument("--tower-loss-mode", choices=["bcd", "all"], default="bcd",
help="Main-phase tower loss strategy: "
"'bcd' (Block Coordinate Descent — randomly train one tower or fused per step) "
"or 'all' (sum all three losses — fused + img + md — every step).")
ap.add_argument("--backbone", default="refugelike")
ap.add_argument("--freeze-ratio", type=float, default=0.0)
ap.add_argument("--augment", action="store_true")
ap.add_argument("--balanced-sampling", action="store_true",
help="Use WeightedRandomSampler during training to equalise class frequency (default: off).")
ap.add_argument("--num-workers", type=int, default=0)
ap.add_argument("--num-workers", type=int, default=4)
ap.add_argument("--in-memory-cache", action="store_true", default=True,
help="Cache preprocessed images in RAM (default: on).")
ap.add_argument("--no-in-memory-cache", action="store_false", dest="in_memory_cache",
help="Disable in-memory image cache.")
ap.add_argument("--cache-workers", type=int, default=4,
help="Threads for prebuilding in-memory image cache (default: 4).")
ap.add_argument("--device", choices=["auto", "cpu", "cuda"], default="auto")
ap.add_argument("--seed", type=int, default=1234)
ap.add_argument("--run-name", default=None)
@@ -289,6 +305,21 @@ class V2HyperTower:
ap.add_argument("--log-every", type=int, default=1)
ap.add_argument("--save-checkpoints", action=argparse.BooleanOptionalAction, default=True,
help="Save best_single.pt / best_holdout_single.pt per fold (use --no-save-checkpoints to disable)")
ap.add_argument("--use-last-epoch", action="store_true", default=False,
help="Score using the final epoch's model state rather than the best-AUC checkpoint.")
# IOP feature options
ap.add_argument(
"--iop-corr-method",
choices=["ratio", "ols", "lad", "multi"],
default="ratio",
help="Perkins→Pneumatic conversion method: ratio (default), ols, lad, or multi (+CCT).",
)
ap.add_argument(
"--iop-drop-raw",
action="store_true",
default=False,
help="Exclude IOP_raw from the feature matrix (keep only IOP_corr).",
)
ap.add_argument(
"--fused-head", action="store_true",
help="(ensemble mode only) After base SingleEyeHT training, freeze it and train a "
@@ -319,6 +350,9 @@ class V2HyperTower:
cat_cols=list(args.cat_cols),
n_splits=args.n_splits,
random_seed=args.fold_seed,
iop_corr_method=getattr(args, "iop_corr_method", "ratio"),
iop_drop_raw=getattr(args, "iop_drop_raw", False),
exclude_cols=list(getattr(args, "exclude_cols", []) or []),
)
print(f"Loaded: {len(self.data.df)} rows feature_dim={self.data.feature_dim}", flush=True)
self.image_preprocessor = build_image_preprocessor_from_args(args)
@@ -407,6 +441,49 @@ class V2HyperTower:
patient_col="Patient ID", label_col=args.label_col, sample_mode="patient"
)
# ---- PredictionStore — build once before fold loop ---------------
fused_head = getattr(args, "fused_head", False)
_head_names = head_names_for_mode(tower_mode, fused_head=fused_head)
fusion_epochs = int(getattr(args, "fusion_epochs", 10)) if fused_head else 0
_global_warmup_tower = getattr(args, "warmup_tower_epochs", None)
_global_warmup_fused = getattr(args, "warmup_fused_epochs", None)
_warmup_tower = (
int(args.single_warmup_tower_epochs)
if getattr(args, "single_warmup_tower_epochs", None) is not None
else int(_global_warmup_tower) if _global_warmup_tower is not None else 2
)
_warmup_fused = (
int(args.single_warmup_fused_epochs)
if getattr(args, "single_warmup_fused_epochs", None) is not None
else int(_global_warmup_fused) if _global_warmup_fused is not None else 2
)
_warmup_md = int(getattr(args, "warmup_md_epochs", 0))
_total_epochs = _warmup_md + _warmup_tower + _warmup_fused + int(args.epochs) + fusion_epochs
# sample IDs depend on mode: single uses eye IDs, others use patient IDs
if tower_mode in ("single", "classic"):
_sample_ids = [
f"{row['Patient ID']}{row['eyeID']}"
for _, row in df_mode.iterrows()
]
_y_true = df_mode[args.label_col].tolist()
else:
# one row per patient (deduplicate — take first occurrence per patient)
_pat_df = df_mode.drop_duplicates(subset="Patient ID")
_sample_ids = _pat_df["Patient ID"].astype(str).tolist()
_y_true = _pat_df[args.label_col].tolist()
pred_store = PredictionStore(
sample_ids=_sample_ids,
y_true=_y_true,
head_names=_head_names,
n_folds=n_folds,
n_epochs=_total_epochs,
n_classes=num_classes,
)
image_cache: dict | None = {} if getattr(args, "in_memory_cache", False) else None
for fold in range(n_folds):
seed_everything(args.seed + fold * 100)
fold_dir = tm_dir / f"fold{fold}"
@@ -423,6 +500,8 @@ class V2HyperTower:
profile_patient=profile_patient,
fold_dir=fold_dir,
tower_mode=tower_mode,
pred_store=pred_store,
image_cache=image_cache,
)
fold_results.append(result)
if artifacts.y_true_ensemble is not None:
@@ -433,6 +512,8 @@ class V2HyperTower:
np.save(fold_dir / "probs_img.npy", artifacts.probs_ensemble_img)
if artifacts.probs_ensemble_md is not None:
np.save(fold_dir / "probs_md.npy", artifacts.probs_ensemble_md)
if artifacts.y_true_classic is not None:
np.save(fold_dir / "y_true.npy", artifacts.y_true_classic)
if artifacts.probs_classic is not None:
np.save(fold_dir / "probs_classic.npy", artifacts.probs_classic)
if artifacts.probs_classic_img is not None:
@@ -530,6 +611,7 @@ class V2HyperTower:
"run_id": run_name,
"backbone": args.backbone,
"epochs": args.epochs,
"warmup_md_epochs": getattr(args, "warmup_md_epochs", 0),
"warmup_tower_epochs": args.warmup_tower_epochs,
"warmup_fused_epochs": args.warmup_fused_epochs,
"single_warmup_tower_epochs": args.single_warmup_tower_epochs,
@@ -549,6 +631,7 @@ class V2HyperTower:
"mode_summary": summary,
}
(tm_dir / "summary.json").write_text(json.dumps(mode_summary, indent=2), encoding="utf-8")
pred_store.save(tm_dir / "predictions.npz")
root_summary_path = out_dir / "summary.json"
if root_summary_path.exists():
@@ -583,6 +666,8 @@ class V2HyperTower:
profile_patient,
fold_dir: Path,
tower_mode: str,
pred_store: "PredictionStore | None" = None,
image_cache: "dict | None" = None,
) -> tuple[FoldResult, FoldArtifacts]:
args = self.args
device = self.device
@@ -615,6 +700,7 @@ class V2HyperTower:
if getattr(args, "bilat_warmup_fused_epochs", None) is not None
else int(global_warmup_fused) if global_warmup_fused is not None else 3
)
single_warmup_md = int(getattr(args, "warmup_md_epochs", 0)) if run_single else 0
if not run_single:
single_warmup_tower = 0
single_warmup_fused = 0
@@ -622,7 +708,7 @@ class V2HyperTower:
bilat_warmup_tower = 0
bilat_warmup_fused = 0
main_epochs = int(args.epochs)
total_single_epochs = (single_warmup_tower + single_warmup_fused + main_epochs) if run_single else 0
total_single_epochs = (single_warmup_md + single_warmup_tower + single_warmup_fused + main_epochs) if run_single else 0
total_bilat_epochs = (bilat_warmup_tower + bilat_warmup_fused + main_epochs) if run_bilat else 0
total_epochs = max(total_single_epochs, total_bilat_epochs)
@@ -631,6 +717,18 @@ class V2HyperTower:
bilat_train = filter_bilateral_samples(profile_patient.build_samples(df=split.train, clinical=data))
bilat_val = filter_bilateral_samples(profile_patient.build_samples(df=split.val, clinical=data))
# Register split labels in the prediction store
if pred_store is not None:
if tower_mode in ("single", "classic"):
# eye-level IDs: "{patient_id}{eyeID}"
train_sids = [f"{s['id_1']}{s.get('eye_id_1','')}" for s in eye_train]
val_sids = [f"{s['id_1']}{s.get('eye_id_1','')}" for s in bilat_val]
else:
train_sids = [str(s["id_1"]) for s in bilat_train]
val_sids = [str(s["id_1"]) for s in bilat_val]
pred_store.set_split(fold, train_sids, "train")
pred_store.set_split(fold, val_sids, "val")
if len(bilat_val) == 0:
print(f" [fold {fold+1}] WARNING: no bilateral val samples; skipping fold.", flush=True)
empty = FoldResult(
@@ -681,13 +779,15 @@ class V2HyperTower:
slots_eye = profile_eye.slot_descriptors()
slots_patient = profile_patient.slot_descriptors()
loader_kw = dict(batch_size=args.batch_size, num_workers=args.num_workers)
loader_kw = dict(batch_size=args.batch_size, num_workers=args.num_workers,
image_cache=image_cache)
# ---- loaders ---------------------------------------------------
use_balanced = bool(getattr(args, "balanced_sampling", False))
train_single_loader = None
train_eval_loader = None # non-shuffled, no sampler — for per-epoch train logging
train_bilat_loader = None
md_only_loader = None # image-free loader for md_warmup phase
if run_single:
single_sampler = build_balanced_sampler(eye_train) if use_balanced else None
train_single_loader = make_loader(
@@ -705,6 +805,20 @@ class V2HyperTower:
shuffle=False,
**loader_kw,
)
if single_warmup_md > 0:
# MD-only loader: drop image_1 so PIL never opens files during md_warmup.
# Always use balanced sampling for md_warmup — MD features alone are weaker
# than images and collapse to majority class without class balancing.
slots_md_only = {k: v for k, v in slots_eye.items() if k != "image_1"}
md_warmup_sampler = single_sampler if single_sampler is not None else build_balanced_sampler(eye_train)
md_only_loader = make_loader(
eye_train, slots_md_only,
image_transform=None,
image_preprocessor=None,
shuffle=True,
sampler=md_warmup_sampler,
**loader_kw,
)
if run_bilat:
bilat_sampler = build_balanced_sampler(bilat_train) if use_balanced else None
train_bilat_loader = make_loader(
@@ -751,6 +865,18 @@ class V2HyperTower:
**loader_kw,
)
print(f" [fold {fold+1}] holdout_n={len(holdout_bilat)} (bilateral patients)", flush=True)
if pred_store is not None:
pred_store.set_split(fold, [str(s["id_1"]) for s in holdout_bilat], "holdout")
# ---- prebuild in-memory image cache (fold 0 only; shared dict fills for later folds) ----
if image_cache is not None:
cache_workers = int(getattr(args, "cache_workers", 4))
_loaders_to_warm = [
train_single_loader, train_bilat_loader, val_loader, holdout_loader,
]
for _ldr in _loaders_to_warm:
if _ldr is not None:
_ldr.dataset.prebuild_image_cache(cache_workers=cache_workers)
opt_single = torch.optim.Adam(single.parameters(), lr=args.lr) if run_single else None
opt_bilateral = torch.optim.Adam(bilateral.parameters(), lr=args.lr) if run_bilat else None
@@ -849,7 +975,7 @@ class V2HyperTower:
print(
f" [fold {fold+1}] single_train_n={len(eye_train)} (eye-level) "
f"val_n={len(bilat_val)} "
f"single_warmup={single_warmup_tower}+{single_warmup_fused} total={total_single_epochs}",
f"single_warmup=md{single_warmup_md}+twr{single_warmup_tower}+fus{single_warmup_fused} total={total_single_epochs}",
flush=True,
)
else:
@@ -861,17 +987,21 @@ class V2HyperTower:
)
# ---- epoch loop ------------------------------------------------
_prev_phase_single = "inactive" # used to detect md_warmup → next phase transition
for epoch in range(total_epochs):
_epoch_t0 = time.time()
if not run_single:
phase_single, main_epoch_single, single_active = "inactive", 0, False
elif epoch < single_warmup_tower:
elif epoch < single_warmup_md:
phase_single, main_epoch_single, single_active = "md_warmup", 0, True
elif epoch < (single_warmup_md + single_warmup_tower):
phase_single, main_epoch_single, single_active = "tower_warmup", 0, True
elif epoch < (single_warmup_tower + single_warmup_fused):
elif epoch < (single_warmup_md + single_warmup_tower + single_warmup_fused):
phase_single, main_epoch_single, single_active = "fused_warmup", 0, True
elif epoch < total_single_epochs:
phase_single, main_epoch_single, single_active = (
"main",
epoch - single_warmup_tower - single_warmup_fused + 1,
epoch - single_warmup_md - single_warmup_tower - single_warmup_fused + 1,
True,
)
else:
@@ -893,9 +1023,11 @@ class V2HyperTower:
phase_bilat, main_epoch_bilat, bilat_active = "done", main_epochs, False
if run_single and single_active:
_active_loader = md_only_loader if phase_single == "md_warmup" else train_single_loader
sl_loss, sl_acc = train_single_epoch(
single, train_single_loader, opt_single, device,
single, _active_loader, opt_single, device,
phase=phase_single, bcd_prob=float(args.bcd_prob),
tower_loss_mode=args.tower_loss_mode,
)
else:
sl_loss, sl_acc = nan, nan
@@ -904,11 +1036,14 @@ class V2HyperTower:
bl_loss, bl_acc = train_bilateral_epoch(
bilateral, train_bilat_loader, opt_bilateral, device,
phase=phase_bilat, bcd_prob=float(args.bcd_prob),
tower_loss_mode=args.tower_loss_mode,
)
else:
bl_loss, bl_acc = nan, nan
if run_single and tower_mode == "single":
_skip_val_eval = (phase_single == "md_warmup")
if run_single and tower_mode == "single" and not _skip_val_eval:
y_cl, p_cl, p_cl_img, p_cl_md = collect_probs_single_components(
single, val_loader, device, aggregate_patient=False
)
@@ -918,11 +1053,11 @@ class V2HyperTower:
_, cl_auc_img, _ = _score_arrays(y_cl, p_cl_img, num_classes)
_, cl_auc_md, _ = _score_arrays(y_cl, p_cl_md, num_classes)
y_en = np.array([], dtype=np.int64)
p_en = np.zeros((0, 0), dtype=np.float32)
p_en = p_en_img = p_en_md = np.zeros((0, num_classes), dtype=np.float32)
en_acc = en_auc = nan
en_n = 0
en_acc_img = en_acc_md = en_auc_img = en_auc_md = nan
elif run_single and tower_mode == "ensemble":
elif run_single and tower_mode == "ensemble" and not _skip_val_eval:
(y_en,
_p_en_f_od, _p_en_i_od, _p_en_m_od,
_p_en_f_os, _p_en_i_os, _p_en_m_os,
@@ -939,19 +1074,20 @@ class V2HyperTower:
_, en_auc_img, _ = _score_arrays(y_en, p_en_img, num_classes)
_, en_auc_md, _ = _score_arrays(y_en, p_en_md, num_classes)
y_cl = np.array([], dtype=np.int64)
p_cl = np.zeros((0, 0), dtype=np.float32)
p_cl = p_cl_img = p_cl_md = np.zeros((0, num_classes), dtype=np.float32)
cl_acc = cl_auc = nan
cl_n = 0
cl_acc_img = cl_acc_md = cl_auc_img = cl_auc_md = nan
else:
y_cl = y_en = np.array([], dtype=np.int64)
p_cl = p_en = np.zeros((0, 0), dtype=np.float32)
p_cl = p_cl_img = p_cl_md = np.zeros((0, num_classes), dtype=np.float32)
p_en = p_en_img = p_en_md = np.zeros((0, num_classes), dtype=np.float32)
cl_acc = cl_auc = en_acc = en_auc = nan
cl_n = en_n = 0
cl_acc_img = cl_acc_md = en_acc_img = en_acc_md = nan
cl_auc_img = cl_auc_md = en_auc_img = en_auc_md = nan
if run_bilat:
if run_bilat and not _skip_val_eval:
y_bi, p_bi, p_bi_img, p_bi_md = collect_probs_bilateral_components(
bilateral, val_loader, device
)
@@ -975,7 +1111,7 @@ class V2HyperTower:
p_cl_h = p_cl_h_img = p_cl_h_md = _z2
p_en_h = p_en_h_img = p_en_h_md = _z2
if holdout_loader is not None:
if holdout_loader is not None and not _skip_val_eval:
if run_single and tower_mode == "single":
y_cl_h, p_cl_h, p_cl_h_img, p_cl_h_md = collect_probs_single_components(
single, holdout_loader, device, aggregate_patient=False
@@ -1033,7 +1169,7 @@ class V2HyperTower:
tr_fe_corr = tr_fe_err = tr_n = 0
y_tr = np.array([], dtype=np.int64)
p_tr_f = p_tr_i = p_tr_m = np.zeros((0, num_classes), dtype=np.float32)
if run_single and train_eval_loader is not None:
if run_single and train_eval_loader is not None and not _skip_val_eval:
y_tr, p_tr_f, p_tr_i, p_tr_m, tr_ids = collect_probs_eye_level(
single, train_eval_loader, device, return_ids=True
)
@@ -1052,6 +1188,23 @@ class V2HyperTower:
_epoch_train_pm.append(p_tr_m)
_epoch_train_ids.append(tr_ids)
_epoch_train_y.append(y_tr)
# record into PredictionStore
if pred_store is not None:
if tower_mode in ("single", "classic"):
pred_store.record(fold, epoch, tr_ids, "fused", p_tr_f)
pred_store.record(fold, epoch, tr_ids, "img", p_tr_i)
pred_store.record(fold, epoch, tr_ids, "md", p_tr_m)
else: # ensemble: separate OD and OS by eye suffix
od_mask = np.array([str(i).endswith("OD") for i in tr_ids])
os_mask = ~od_mask
od_pids = [str(i)[:-2] for i in tr_ids[od_mask]]
os_pids = [str(i)[:-2] for i in tr_ids[os_mask]]
pred_store.record(fold, epoch, od_pids, "od_fused", p_tr_f[od_mask])
pred_store.record(fold, epoch, od_pids, "od_img", p_tr_i[od_mask])
pred_store.record(fold, epoch, od_pids, "od_md", p_tr_m[od_mask])
pred_store.record(fold, epoch, os_pids, "os_fused", p_tr_f[os_mask])
pred_store.record(fold, epoch, os_pids, "os_img", p_tr_i[os_mask])
pred_store.record(fold, epoch, os_pids, "os_md", p_tr_m[os_mask])
# accumulate val for npy tensors
if run_single and tower_mode == "ensemble" and y_en.size:
@@ -1073,6 +1226,20 @@ class V2HyperTower:
_epoch_val_pm_os.append(p_cl_md)
_epoch_val_y.append(y_cl)
# record val into PredictionStore
if pred_store is not None:
if run_single and tower_mode == "ensemble" and y_en.size:
pred_store.record(fold, epoch, _en_pat_ids, "od_fused", _p_en_f_od)
pred_store.record(fold, epoch, _en_pat_ids, "od_img", _p_en_i_od)
pred_store.record(fold, epoch, _en_pat_ids, "od_md", _p_en_m_od)
pred_store.record(fold, epoch, _en_pat_ids, "os_fused", _p_en_f_os)
pred_store.record(fold, epoch, _en_pat_ids, "os_img", _p_en_i_os)
pred_store.record(fold, epoch, _en_pat_ids, "os_md", _p_en_m_os)
elif run_single and tower_mode == "single" and y_cl.size:
# val in single mode: collect_probs_single_components(aggregate_patient=False)
# returns interleaved [all_OD, all_OS] per batch — IDs not tracked here yet
pass # single-mode val IDs not currently available; train IDs are sufficient
# Best-epoch checks (restricted to main phase).
target_single_auc = cl_auc if tower_mode == "single" else en_auc
target_holdout_single_auc = cl_auc_h if tower_mode == "single" else en_auc_h
@@ -1194,14 +1361,54 @@ class V2HyperTower:
**cm_row,
}, optional_cols=epoch_fields)
# ---- md_warmup progress bar (replaces per-epoch print) --------
if phase_single == "md_warmup":
_bar_w = 30
_filled = int(_bar_w * (epoch + 1) / single_warmup_md)
_bar = "#" * _filled + "-" * (_bar_w - _filled)
_bar_msg = (
f" [fold {fold+1}] md_warmup [{_bar}] "
f"{epoch + 1}/{single_warmup_md} loss={sl_loss:.4f}"
)
print(f"\r{_bar_msg}", end="", flush=True)
fold_logger.info(_bar_msg)
_prev_phase_single = phase_single
continue # skip normal log block entirely
if _prev_phase_single == "md_warmup":
print() # seal the progress bar line
if args.log_every > 0 and (epoch + 1) % args.log_every == 0:
_epoch_secs = time.time() - _epoch_t0
hld_auc = target_holdout_single_auc if run_single else bi_auc_h
hld_suffix = f" hld_auc={hld_auc:.4f}" if holdout_loader is not None else ""
# Human-readable phase progress for console logs.
if phase_single == "tower_warmup":
single_phase_epoch = epoch - single_warmup_md + 1
single_phase_total = single_warmup_tower
elif phase_single == "fused_warmup":
single_phase_epoch = epoch - single_warmup_md - single_warmup_tower + 1
single_phase_total = single_warmup_fused
else:
single_phase_epoch = main_epoch_single
single_phase_total = main_epochs
if phase_bilat == "tower_warmup":
bilat_phase_epoch = epoch + 1
bilat_phase_total = bilat_warmup_tower
elif phase_bilat == "fused_warmup":
bilat_phase_epoch = epoch - bilat_warmup_tower + 1
bilat_phase_total = bilat_warmup_fused
else:
bilat_phase_epoch = main_epoch_bilat
bilat_phase_total = main_epochs
if run_single:
if tower_mode == "single":
msg = (
f" ep {epoch+1:>3}/{total_epochs} "
f"[single:{phase_single} {main_epoch_single}/{main_epochs}] "
f" ep {epoch+1:>3}/{total_epochs} ({_epoch_secs:.1f}s) "
f"[single:{phase_single} {single_phase_epoch}/{single_phase_total}] "
f"fused(acc={cl_acc:.4f},auc={cl_auc:.4f}) "
f"img(acc={cl_acc_img:.4f},auc={cl_auc_img:.4f}) "
f"md(acc={cl_acc_md:.4f},auc={cl_auc_md:.4f}) "
@@ -1210,8 +1417,8 @@ class V2HyperTower:
)
else:
msg = (
f" ep {epoch+1:>3}/{total_epochs} "
f"[single:{phase_single} {main_epoch_single}/{main_epochs}] "
f" ep {epoch+1:>3}/{total_epochs} ({_epoch_secs:.1f}s) "
f"[single:{phase_single} {single_phase_epoch}/{single_phase_total}] "
f"fused(acc={en_acc:.4f},auc={en_auc:.4f}) "
f"img(acc={en_acc_img:.4f},auc={en_auc_img:.4f}) "
f"md(acc={en_acc_md:.4f},auc={en_auc_md:.4f}) "
@@ -1220,8 +1427,8 @@ class V2HyperTower:
)
else:
msg = (
f" ep {epoch+1:>3}/{total_epochs} "
f"[bilat:{phase_bilat} {main_epoch_bilat}/{main_epochs}] "
f" ep {epoch+1:>3}/{total_epochs} ({_epoch_secs:.1f}s) "
f"[bilat:{phase_bilat} {bilat_phase_epoch}/{bilat_phase_total}] "
f"fused(acc={bi_acc:.4f},auc={bi_auc:.4f}) "
f"img(acc={bi_acc_img:.4f},auc={bi_auc_img:.4f}) "
f"md(acc={bi_acc_md:.4f},auc={bi_auc_md:.4f}) "
@@ -1231,8 +1438,16 @@ class V2HyperTower:
print(msg, flush=True)
fold_logger.info(msg)
_prev_phase_single = phase_single
fold_logger.close()
# Override: use final epoch state instead of best-AUC checkpoint
if getattr(args, "use_last_epoch", False):
best_single_state = copy.deepcopy(single.state_dict())
if run_bilat:
best_bilat_state = copy.deepcopy(bilat.state_dict())
if args.save_checkpoints:
if best_single_state is not None:
torch.save(best_single_state, fold_dir / "best_single.pt")
@@ -1285,10 +1500,14 @@ class V2HyperTower:
flush=True,
)
_val_pids_for_store = [str(s["id_1"]) for s in bilat_val]
for fep in range(fusion_epochs):
fu_loss, fu_acc = train_fusion_epoch(fused, train_bilat_loader, opt_fused, device)
y_fu, p_fu = collect_probs_fused(fused, val_loader, device)
fu_auc = _score_arrays(y_fu, p_fu, num_classes)[1]
if pred_store is not None and y_fu.size:
_store_ep = total_single_epochs + fep
pred_store.record(fold, _store_ep, _val_pids_for_store, "bilat_fused", p_fu)
# Holdout eval (if available)
fu_hld_auc = nan
environment.yml
+216
View File
@@ -0,0 +1,216 @@
name: fundus_imaging
channels:
- conda-forge
- defaults
dependencies:
- _libgcc_mutex=0.1=conda_forge
- _openmp_mutex=4.5=2_gnu
- alsa-lib=1.2.11=hd590300_1
- asttokens=2.4.1=pyhd8ed1ab_0
- attr=2.5.1=h166bdaf_1
- blas=1.0=openblas
- bottleneck=1.3.7=py312ha883a20_0
- brotli=1.0.9=h5eee18b_8
- brotli-bin=1.0.9=h5eee18b_8
- bzip2=1.0.8=hd590300_5
- ca-certificates=2025.9.9=h06a4308_0
- cairo=1.18.0=h3faef2a_0
- comm=0.2.2=pyhd8ed1ab_0
- contourpy=1.2.0=py312hdb19cb5_0
- cycler=0.11.0=pyhd3eb1b0_0
- dbus=1.13.18=hb2f20db_0
- debugpy=1.8.1=py312h30efb56_0
- decorator=5.1.1=pyhd8ed1ab_0
- exceptiongroup=1.2.0=pyhd8ed1ab_2
- executing=2.0.1=pyhd8ed1ab_0
- expat=2.6.2=h6a678d5_0
- font-ttf-dejavu-sans-mono=2.37=hd3eb1b0_0
- font-ttf-inconsolata=2.001=hcb22688_0
- font-ttf-source-code-pro=2.030=hd3eb1b0_0
- font-ttf-ubuntu=0.83=h8b1ccd4_0
- fontconfig=2.14.2=h14ed4e7_0
- fonts-anaconda=1=h8fa9717_0
- fonts-conda-ecosystem=1=hd3eb1b0_0
- fonttools=4.51.0=py312h5eee18b_0
- freetype=2.12.1=h4a9f257_0
- gettext=0.22.5=h59595ed_2
- gettext-tools=0.22.5=h59595ed_2
- glib=2.80.2=hf974151_0
- glib-tools=2.80.2=hb6ce0ca_0
- graphite2=1.3.14=h295c915_1
- gst-plugins-base=1.14.1=h6a678d5_1
- gstreamer=1.14.1=h5eee18b_1
- harfbuzz=8.5.0=hfac3d4d_0
- icu=73.2=h59595ed_0
- imageio=2.37.0=py312h06a4308_0
- importlib-metadata=7.1.0=pyha770c72_0
- importlib_metadata=7.1.0=hd8ed1ab_0
- ipykernel=6.29.3=pyhd33586a_0
- ipython=8.24.0=pyh707e725_0
- ipywidgets=8.1.2=pyhd8ed1ab_1
- jedi=0.19.1=pyhd8ed1ab_0
- joblib=1.4.0=py312h06a4308_0
- jpeg=9e=h5eee18b_1
- jupyter_client=8.6.1=pyhd8ed1ab_0
- jupyter_core=5.7.2=py312h7900ff3_0
- jupyterlab_widgets=3.0.10=py312h06a4308_0
- keyutils=1.6.1=h166bdaf_0
- kiwisolver=1.4.4=py312h6a678d5_0
- krb5=1.20.1=h81ceb04_0
- lame=3.100=h7b6447c_0
- lazy_loader=0.4=py312h06a4308_0
- lcms2=2.12=h3be6417_0
- ld_impl_linux-64=2.40=h55db66e_0
- lerc=3.0=h295c915_0
- libasprintf=0.22.5=h661eb56_2
- libasprintf-devel=0.22.5=h661eb56_2
- libbrotlicommon=1.0.9=h5eee18b_8
- libbrotlidec=1.0.9=h5eee18b_8
- libbrotlienc=1.0.9=h5eee18b_8
- libcap=2.69=h0f662aa_0
- libclang=14.0.6=default_hc6dbbc7_1
- libclang-cpp15=15.0.7=default_h127d8a8_5
- libclang13=14.0.6=default_he11475f_1
- libcups=2.4.2=h2d74bed_1
- libdeflate=1.17=h5eee18b_1
- libedit=3.1.20191231=he28a2e2_2
- libevent=2.1.12=hdbd6064_1
- libexpat=2.6.2=h59595ed_0
- libffi=3.4.2=h7f98852_5
- libflac=1.4.3=h59595ed_0
- libgcc-ng=13.2.0=h77fa898_7
- libgcrypt=1.10.3=hd590300_0
- libgettextpo=0.22.5=h59595ed_2
- libgettextpo-devel=0.22.5=h59595ed_2
- libgfortran=3.0.0=1
- libgfortran-ng=11.2.0=h00389a5_1
- libgfortran5=11.2.0=h1234567_1
- libglib=2.80.2=hf974151_0
- libgomp=13.2.0=h77fa898_7
- libgpg-error=1.49=h4f305b6_0
- libiconv=1.17=hd590300_2
- libjpeg-turbo=2.1.4=h166bdaf_0
- libllvm14=14.0.6=hdb19cb5_3
- libllvm15=15.0.7=hb3ce162_4
- libllvm18=18.1.5=hb77312f_0
- libnsl=2.0.1=hd590300_0
- libogg=1.3.5=h27cfd23_1
- libopenblas=0.3.21=h043d6bf_0
- libopus=1.3.1=h7b6447c_0
- libpng=1.6.43=h2797004_0
- libpq=12.17=hdbd6064_0
- libsndfile=1.2.2=hc60ed4a_1
- libsodium=1.0.18=h36c2ea0_1
- libsqlite=3.45.3=h2797004_0
- libstdcxx-ng=13.2.0=hc0a3c3a_7
- libsystemd0=255=h3516f8a_1
- libtiff=4.5.1=h6a678d5_0
- libuuid=2.38.1=h0b41bf4_0
- libvorbis=1.3.7=h7b6447c_0
- libwebp-base=1.3.2=h5eee18b_0
- libxcb=1.15=h7f8727e_0
- libxcrypt=4.4.36=hd590300_1
- libxkbcommon=1.7.0=h662e7e4_0
- libxml2=2.12.7=hc051c1a_0
- libzlib=1.2.13=hd590300_5
- lz4-c=1.9.4=h6a678d5_1
- matplotlib=3.8.4=py312h06a4308_0
- matplotlib-base=3.8.4=py312h526ad5a_0
- matplotlib-inline=0.1.7=pyhd8ed1ab_0
- mpg123=1.32.6=h59595ed_0
- mysql=5.7.20=hf484d3e_1001
- mysql-common=8.3.0=hf1915f5_4
- mysql-libs=8.3.0=hca2cd23_4
- ncurses=6.5=h59595ed_0
- nest-asyncio=1.6.0=pyhd8ed1ab_0
- networkx=3.4.2=py312h06a4308_0
- nspr=4.35=h6a678d5_0
- nss=3.100=hca3bf56_0
- numexpr=2.8.7=py312he7dcb8a_0
- numpy=1.26.4=py312h2809609_0
- numpy-base=1.26.4=py312he1a6c75_0
- openblas=0.3.4=ha44fe06_0
- openjpeg=2.4.0=h3ad879b_0
- openssl=3.3.0=hd590300_0
- packaging=24.0=pyhd8ed1ab_0
- pandas=2.2.1=py312h526ad5a_0
- parso=0.8.4=pyhd8ed1ab_0
- pcre2=10.43=hcad00b1_0
- pexpect=4.9.0=pyhd8ed1ab_0
- pickleshare=0.7.5=py_1003
- pillow=10.3.0=py312h5eee18b_0
- pixman=0.43.2=h59595ed_0
- platformdirs=4.2.1=pyhd8ed1ab_0
- ply=3.11=py312h06a4308_1
- prompt-toolkit=3.0.42=pyha770c72_0
- psutil=5.9.8=py312h98912ed_0
- ptyprocess=0.7.0=pyhd3deb0d_0
- pulseaudio-client=17.0=hb77b528_0
- pure_eval=0.2.2=pyhd8ed1ab_0
- pybind11-abi=5=hd3eb1b0_0
- pygments=2.18.0=pyhd8ed1ab_0
- pyparsing=3.0.9=py312h06a4308_0
- pyqt=5.15.10=py312h6a678d5_0
- pyqt5-sip=12.13.0=py312h5eee18b_0
- python=3.12.3=hab00c5b_0_cpython
- python-dateutil=2.9.0=pyhd8ed1ab_0
- python-tzdata=2023.3=pyhd3eb1b0_0
- python_abi=3.12=4_cp312
- pytz=2024.1=py312h06a4308_0
- pyzmq=26.0.3=py312h8fd38d8_0
- qt-main=5.15.2=h53bd1ea_10
- readline=8.2=h8228510_1
- scikit-image=0.25.2=py312hc74f9fe_0
- scikit-learn=1.4.2=py312h526ad5a_1
- scipy=1.13.0=py312h2809609_0
- setuptools=69.5.1=pyhd8ed1ab_0
- sip=6.7.12=py312h6a678d5_0
- six=1.16.0=pyh6c4a22f_0
- sqlite=3.45.3=h5eee18b_0
- stack_data=0.6.2=pyhd8ed1ab_0
- threadpoolctl=2.2.0=pyh0d69192_0
- tifffile=2024.12.12=py312h06a4308_0
- tk=8.6.13=noxft_h4845f30_101
- tornado=6.4=py312h98912ed_0
- traitlets=5.14.3=pyhd8ed1ab_0
- typing_extensions=4.11.0=pyha770c72_0
- tzdata=2024a=h0c530f3_0
- unicodedata2=15.1.0=py312h5eee18b_0
- wcwidth=0.2.13=pyhd8ed1ab_0
- widgetsnbextension=4.0.10=py312h06a4308_0
- xcb-util=0.4.0=hd590300_1
- xcb-util-image=0.4.0=h8ee46fc_1
- xcb-util-keysyms=0.4.0=h8ee46fc_1
- xcb-util-renderutil=0.3.9=hd590300_1
- xcb-util-wm=0.4.1=h8ee46fc_1
- xkeyboard-config=2.41=hd590300_0
- xlrd=2.0.1=pyhd3eb1b0_1
- xorg-kbproto=1.0.7=h7f98852_1002
- xorg-libice=1.1.1=hd590300_0
- xorg-libsm=1.2.4=h7391055_0
- xorg-libx11=1.8.9=h8ee46fc_0
- xorg-libxau=1.0.11=hd590300_0
- xorg-libxext=1.3.4=h0b41bf4_2
- xorg-libxrender=0.9.11=hd590300_0
- xorg-renderproto=0.11.1=h7f98852_1002
- xorg-xextproto=7.3.0=h0b41bf4_1003
- xorg-xf86vidmodeproto=2.3.1=h7f98852_1002
- xorg-xproto=7.0.31=h27cfd23_1007
- xz=5.4.6=h5eee18b_1
- zeromq=4.3.5=h6a678d5_0
- zipp=3.17.0=pyhd8ed1ab_0
- zlib=1.2.13=hd590300_5
- zstd=1.5.6=ha6fb4c9_0
- pip:
- et-xmlfile==2.0.0
- opencv-python==4.9.0.80
- openpyxl==3.1.5
- pip==25.2
- pytorch-triton-rocm==3.2.0+rocm6.4.1.git6da9e660
- sympy==1.13.1
- torch==2.6.0+rocm6.4.1.git1ded221d
- torchaudio==2.6.0+rocm6.4.1.gitd8831425
- torchvision==0.21.0+rocm6.4.1.git4040d51f
- tqdm==4.66.4
- wheel==0.45.1
prefix: /home/rpotter/miniconda3/envs/fundus_imaging
models/refuge/classifier/densenet/densenetrefuge.pt
BIN
View File
Binary file not shown.
models/refuge/classifier/refuge_densenet_backbone.pt
BIN
View File
Binary file not shown.
models/v2/refuge/segmentation/per_image_refuge_build/best.pt
BIN
View File
Binary file not shown.
portable_versions/image_loader.py
+351
View File
@@ -0,0 +1,351 @@
"""
portable_versions/image_loader.py
==================================
A self-contained image loader with in-memory caching, an optional
preprocessing pipeline (e.g. disc cropping), and composable augmentations.
Returns plain NumPy arrays — works with PyTorch, TensorFlow, JAX, or
anything else that can consume an ndarray.
Dependencies: Pillow, numpy (nothing else)
Quickstart
----------
from portable_versions.image_loader import ImageLoader, RandomHorizontalFlip, RandomRotation, ColorJitter
# 1. Build the loader (once per run)
loader = ImageLoader(
target_size=(200, 200),
normalize=True, # float32 in [0, 1] with ImageNet mean/std
cache=True, # each image decoded from disk only once
workers=4, # parallel cache warm-up threads
preprocessor=my_crop_fn, # optional callable(PIL.Image) -> PIL.Image
)
# 2. Attach augmentations (applied randomly and independently per call)
loader.augmentation = [
RandomHorizontalFlip(p=0.5),
RandomRotation(degrees=15),
ColorJitter(brightness=0.2, contrast=0.2, saturation=0.1, hue=0.05),
]
# 3. Warm the cache up front (optional but fast)
loader.warm(all_paths)
# 4. Fetch images by path list — call as many times as you like
# Returns ndarray of shape (N, H, W, 3), dtype float32
imgs = loader.get_img(train_paths)
# For TensorFlow:
import tensorflow as tf
tensor = tf.constant(imgs) # (N, H, W, 3)
# For PyTorch:
import torch
tensor = torch.from_numpy(imgs).permute(0, 3, 1, 2) # (N, C, H, W)
Augmentations reference
-----------------------
All augmentation classes live in this file and depend only on PIL + numpy.
RandomHorizontalFlip(p=0.5)
RandomVerticalFlip(p=0.5)
RandomRotation(degrees=15)
ColorJitter(brightness=0.2, contrast=0.2, saturation=0.1, hue=0.05)
RandomGrayscale(p=0.1)
You can also pass any callable(PIL.Image.Image) -> PIL.Image.Image as an
augmentation step.
"""
from __future__ import annotations
import random
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Callable, Iterable, List, Optional, Tuple, Union
import numpy as np
from PIL import Image, ImageEnhance, ImageOps
# ImageNet channel statistics (RGB)
_IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
_IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)
PathLike = Union[str, Path]
def _call_preprocessor(
fn: Callable[..., Image.Image],
img: Image.Image,
path: Path,
) -> Image.Image:
"""Call preprocessor as fn(img, path) if it accepts two args, else fn(img)."""
try:
return fn(img, path)
except TypeError:
return fn(img)
# ---------------------------------------------------------------------------
# Core loader
# ---------------------------------------------------------------------------
class ImageLoader:
"""
Preprocessing pipeline + in-memory cache + augmentation, returning NumPy.
Parameters
----------
target_size : (height, width)
Output spatial dimensions. Applied after ``preprocessor`` (if any).
Ignored when a ``preprocessor`` already resizes to the right size.
normalize : bool
When True, output is float32 with ImageNet mean/std subtraction.
When False, output is uint8 in [0, 255].
cache : bool
Store decoded+preprocessed images in RAM so each file is read from
disk at most once. The cache persists across ``get_img`` calls.
workers : int
Thread count for ``warm()``. 0 or 1 = single-threaded.
preprocessor : callable, optional
Called as ``preprocessor(img: PIL.Image) -> PIL.Image`` before
resizing and caching. Use this for disc cropping, padding, etc.
augmentation : list of callables
Each element is called as ``fn(img: PIL.Image) -> PIL.Image``.
Applied **after** cache retrieval, so augmentations are NOT cached —
they are re-sampled independently on every ``get_img`` call.
"""
def __init__(
self,
target_size: Tuple[int, int] = (224, 224),
*,
normalize: bool = True,
cache: bool = True,
workers: int = 4,
preprocessor: Optional[Callable[[Image.Image], Image.Image]] = None,
) -> None:
self.target_size = target_size
self.normalize = normalize
self.workers = workers
self.preprocessor = preprocessor
self.augmentation: List[Callable[[Image.Image], Image.Image]] = []
self._cache: Optional[dict[str, np.ndarray]] = {} if cache else None
self._lock = threading.Lock()
# ------------------------------------------------------------------
# Public
# ------------------------------------------------------------------
def warm(self, paths: Iterable[PathLike]) -> None:
"""
Pre-load all *paths* into the cache in parallel.
Already-cached paths are skipped, so calling ``warm`` multiple
times (e.g. once per fold) is safe and only loads new images.
"""
if self._cache is None:
return
paths = [str(p) for p in paths]
to_warm = [p for p in paths if p not in self._cache]
if not to_warm:
return
already = len(paths) - len(to_warm)
print(
f"[ImageLoader] warming {len(to_warm)} images"
+ (f" ({already} already cached)" if already else ""),
flush=True,
)
def _load_one(path_str: str) -> None:
arr = self._decode(path_str)
with self._lock:
self._cache.setdefault(path_str, arr)
if self.workers <= 1:
for p in to_warm:
_load_one(p)
else:
with ThreadPoolExecutor(max_workers=self.workers) as ex:
futures = {ex.submit(_load_one, p): p for p in to_warm}
for fut in as_completed(futures):
fut.result()
def get_img(
self,
paths: Iterable[PathLike],
augment: bool = True,
) -> np.ndarray:
"""
Return images for the given paths as a single NumPy array.
Parameters
----------
paths : iterable of path-like
File paths to load. If the cache is enabled and a path has
been warmed (or loaded before), it is served from RAM.
augment : bool
Apply ``self.augmentation`` pipeline. Set to False at eval time.
Returns
-------
np.ndarray, shape (N, H, W, 3)
float32 in [0, 1] (or normalised) if ``self.normalize`` is True,
otherwise uint8 in [0, 255].
"""
imgs = []
for p in paths:
img = self._get_one(str(p), augment=augment)
imgs.append(img)
return np.stack(imgs, axis=0)
# ------------------------------------------------------------------
# Internal
# ------------------------------------------------------------------
def _decode(self, path_str: str) -> np.ndarray:
"""Open, preprocess, and resize → uint8 HWC ndarray (for the cache)."""
img = Image.open(path_str).convert("RGB")
if self.preprocessor is not None:
img = _call_preprocessor(self.preprocessor, img, Path(path_str))
# Only resize here if preprocessor didn't already produce target_size
if img.size != (self.target_size[1], self.target_size[0]):
img = img.resize((self.target_size[1], self.target_size[0]), Image.BILINEAR)
return np.asarray(img, dtype=np.uint8)
def _get_one(self, path_str: str, augment: bool) -> np.ndarray:
if self._cache is not None:
arr = self._cache.get(path_str)
if arr is None:
arr = self._decode(path_str)
with self._lock:
self._cache.setdefault(path_str, arr)
img = Image.fromarray(arr, mode="RGB")
else:
img = Image.open(path_str).convert("RGB")
if self.preprocessor is not None:
img = self.preprocessor(img)
if img.size != (self.target_size[1], self.target_size[0]):
img = img.resize((self.target_size[1], self.target_size[0]), Image.BILINEAR)
if augment and self.augmentation:
for fn in self.augmentation:
img = fn(img)
arr = np.asarray(img, dtype=np.float32) / 255.0
if self.normalize:
arr = (arr - _IMAGENET_MEAN) / _IMAGENET_STD
else:
arr = (arr * 255).clip(0, 255).astype(np.uint8)
return arr
def __len__(self) -> int:
"""Number of images currently in the cache."""
return len(self._cache) if self._cache is not None else 0
def __repr__(self) -> str:
return (
f"ImageLoader(target_size={self.target_size}, "
f"normalize={self.normalize}, "
f"cached={len(self)}, "
f"augmentations={len(self.augmentation)})"
)
# ---------------------------------------------------------------------------
# Augmentation primitives (PIL-only, no torch/tf dependencies)
# ---------------------------------------------------------------------------
class RandomHorizontalFlip:
"""Flip image left-right with probability *p*."""
def __init__(self, p: float = 0.5):
self.p = p
def __call__(self, img: Image.Image) -> Image.Image:
return ImageOps.mirror(img) if random.random() < self.p else img
class RandomVerticalFlip:
"""Flip image top-bottom with probability *p*."""
def __init__(self, p: float = 0.5):
self.p = p
def __call__(self, img: Image.Image) -> Image.Image:
return ImageOps.flip(img) if random.random() < self.p else img
class RandomRotation:
"""Rotate by a uniformly-sampled angle in [-degrees, +degrees]."""
def __init__(self, degrees: float = 15):
self.degrees = degrees
def __call__(self, img: Image.Image) -> Image.Image:
angle = random.uniform(-self.degrees, self.degrees)
return img.rotate(angle, resample=Image.BILINEAR, expand=False)
class ColorJitter:
"""
Randomly jitter brightness, contrast, saturation, and hue.
Each factor is sampled uniformly from [1 - amount, 1 + amount].
Hue shift is sampled from [-hue, +hue] (range 0–0.5).
Pass 0 for any channel to leave it unchanged.
"""
def __init__(
self,
brightness: float = 0.2,
contrast: float = 0.2,
saturation: float = 0.1,
hue: float = 0.05,
):
self.brightness = brightness
self.contrast = contrast
self.saturation = saturation
self.hue = hue
def __call__(self, img: Image.Image) -> Image.Image:
ops = []
if self.brightness:
ops.append(("brightness", self.brightness))
if self.contrast:
ops.append(("contrast", self.contrast))
if self.saturation:
ops.append(("saturation", self.saturation))
if self.hue:
ops.append(("hue", self.hue))
random.shuffle(ops)
for kind, amount in ops:
factor = random.uniform(1 - amount, 1 + amount)
if kind == "brightness":
img = ImageEnhance.Brightness(img).enhance(factor)
elif kind == "contrast":
img = ImageEnhance.Contrast(img).enhance(factor)
elif kind == "saturation":
img = ImageEnhance.Color(img).enhance(factor)
elif kind == "hue":
# PIL has no direct hue enhancer — shift via HSV in numpy
arr = np.asarray(img.convert("HSV"), dtype=np.int16)
shift = int(random.uniform(-self.hue, self.hue) * 255)
arr[:, :, 0] = (arr[:, :, 0] + shift) % 256
img = Image.fromarray(arr.astype(np.uint8), mode="HSV").convert("RGB")
return img
class RandomGrayscale:
"""Convert to grayscale (keeping 3 channels) with probability *p*."""
def __init__(self, p: float = 0.1):
self.p = p
def __call__(self, img: Image.Image) -> Image.Image:
if random.random() < self.p:
img = ImageOps.grayscale(img).convert("RGB")
return img
portable_versions/refuge_mask_adapter.py
+270
View File
@@ -0,0 +1,270 @@
"""
portable_versions/refuge_mask_adapter.py
=========================================
Optic-disc cropper for REFUGE (and REFUGE2) fundus images, designed as a
drop-in ``preprocessor`` for ``ImageLoader``.
Given an image and its corresponding segmentation mask, it:
1. Extracts the optic disc region from the mask
2. Computes a padded bounding box around it
3. Crops and resizes the original image
Dependencies: Pillow, numpy (nothing else)
Quickstart
----------
from portable_versions.image_loader import ImageLoader, RandomHorizontalFlip, RandomRotation, ColorJitter
from portable_versions.refuge_mask_adapter import RefugeMaskCropper
cropper = RefugeMaskCropper(
mask_dir="REFUGE/Annotations/Training400/Disc_Cup_Masks",
scale=1.5, # context around disc (1.0 = tight, 2.0 = lots of context)
target_size=(200, 200), # output size — should match ImageLoader target_size
mask_suffix=".bmp", # REFUGE1 uses .bmp; REFUGE2 uses .png
)
loader = ImageLoader(
target_size=(200, 200),
normalize=True,
preprocessor=cropper,
)
loader.augmentation = [
RandomHorizontalFlip(),
RandomRotation(15),
ColorJitter(0.2, 0.2, 0.1, 0.05),
]
imgs = loader.get_img(image_paths, augment=True) # (N, 200, 200, 3)
REFUGE mask formats
-------------------
REFUGE1 Grayscale BMP: background=128, disc=255, cup=0
REFUGE2 RGB PNG: background detected from image borders, disc/cup by colour
Both are handled automatically.
Directory structure assumption
------------------------------
The cropper looks for the mask with the same stem as the image file, inside
``mask_dir``. If your layout differs, pass a custom ``mask_path_fn``:
cropper = RefugeMaskCropper(
mask_path_fn=lambda img_path: img_path.with_suffix(".bmp"),
scale=1.5,
target_size=(200, 200),
)
"""
from __future__ import annotations
from collections import Counter
from pathlib import Path
from typing import Optional, Tuple
import numpy as np
from PIL import Image
_MASK_DIR_NAMES = {"Disc_Cup_Masks", "Disc_Masks", "Disc_Mask"}
_MASK_SUFFIXES = {".bmp", ".png"}
class RefugeMaskCropper:
"""
Crop a fundus image to the optic disc region using its segmentation mask.
Pass the REFUGE root directory and the cropper will automatically index
all masks underneath it — no need to specify which subdirectory or
file extension.
cropper = RefugeMaskCropper("REFUGE/", scale=1.5)
loader = ImageLoader(target_size=(200, 200), preprocessor=cropper)
imgs = loader.get_img(test_set) # test_set = any list of image paths
Parameters
----------
refuge_root : str or Path
Top-level REFUGE directory. All mask files under directories named
``Disc_Cup_Masks``, ``Disc_Masks``, or ``Disc_Mask`` are indexed
automatically (supports both .bmp and .png).
scale : float
Padding multiplier applied to the disc radius.
1.0 = tight crop, 1.5 = moderate context, 2.5 = lots of context.
target_size : (height, width)
Output size after cropping. Should match ``ImageLoader.target_size``.
"""
def __init__(
self,
refuge_root: str | Path,
*,
scale: float = 1.5,
target_size: Tuple[int, int] = (200, 200),
) -> None:
self.refuge_root = Path(refuge_root)
self.scale = scale
self.target_size = target_size
self._index: dict[str, list[Path]] = {}
self._build_index()
def _build_index(self) -> None:
"""Walk refuge_root and index all mask files by stem (stem → [paths])."""
for mask_dir in self.refuge_root.rglob("*"):
if mask_dir.is_dir() and mask_dir.name in _MASK_DIR_NAMES:
for f in mask_dir.rglob("*"):
if f.is_file() and f.suffix.lower() in _MASK_SUFFIXES:
self._index.setdefault(f.stem, []).append(f)
if not self._index:
raise FileNotFoundError(
f"No mask files found under {self.refuge_root!r}. "
f"Expected directories named: {_MASK_DIR_NAMES}"
)
n_masks = sum(len(v) for v in self._index.values())
print(f"[RefugeMaskCropper] indexed {n_masks} masks ({len(self._index)} unique stems)", flush=True)
# ------------------------------------------------------------------
# Callable interface — drop-in preprocessor for ImageLoader
# ------------------------------------------------------------------
def __call__(
self,
img: Image.Image,
img_path: Optional[str | Path] = None,
) -> Image.Image:
stem = Path(img_path).stem if img_path else None
mask_path = self._lookup(stem, img_path)
disc_mask = _load_disc_mask(mask_path, img.size)
box = _mask_to_crop_box(disc_mask, scale=self.scale, img_size=img.size)
cropped = img.crop(box)
return cropped.resize(
(self.target_size[1], self.target_size[0]), Image.Resampling.BILINEAR
)
def _lookup(self, stem: Optional[str], img_path: Optional[str | Path] = None) -> Path:
if stem is None:
raise ValueError("img_path is required to match the mask.")
candidates = self._index.get(stem)
if not candidates:
raise KeyError(
f"No mask found for image stem {stem!r}. "
f"Available stems (sample): {list(self._index)[:5]}"
)
if len(candidates) == 1:
return candidates[0]
# Pick the mask whose directory components best overlap with img_path
# (ignores the filename itself to handle extension differences)
img_parts = set(Path(img_path).parent.parts) if img_path else set()
return max(candidates, key=lambda m: len(set(m.parent.parts) & img_parts))
def __repr__(self) -> str:
return (
f"RefugeMaskCropper(refuge_root={str(self.refuge_root)!r}, "
f"scale={self.scale}, target_size={self.target_size}, "
f"masks_indexed={len(self._index)})"
)
# ---------------------------------------------------------------------------
# Mask parsing
# ---------------------------------------------------------------------------
def _load_disc_mask(mask_path: Path, img_size: Tuple[int, int]) -> np.ndarray:
"""
Return a binary disc mask (uint8, 1=disc) from a REFUGE mask file.
Handles:
- Grayscale BMP (REFUGE1): background≈128, disc=255, cup=0
- RGB PNG (REFUGE2): background detected from image borders
"""
mask_img = Image.open(mask_path)
if mask_img.mode == "L" or mask_img.mode == "P":
arr = np.asarray(mask_img.convert("L"), dtype=np.uint8)
bg = _border_mode(arr)
disc_mask = (arr != bg).astype(np.uint8)
else:
arr = np.asarray(mask_img.convert("RGB"), dtype=np.uint8)
bg = _border_mode_rgb(arr)
# disc = any non-background pixel
bg_mask = np.all(arr == bg, axis=2)
disc_mask = (~bg_mask).astype(np.uint8)
# Ensure mask matches image spatial size
mh, mw = disc_mask.shape
iw, ih = img_size
if (mw, mh) != (iw, ih):
disc_img = Image.fromarray(disc_mask * 255).resize((iw, ih), Image.NEAREST)
disc_mask = (np.asarray(disc_img) > 0).astype(np.uint8)
return disc_mask
def _border_mode(arr: np.ndarray, border: int = 5) -> int:
"""Most common pixel value along the image border (grayscale)."""
h, w = arr.shape
border_pixels = np.concatenate([
arr[:border, :].ravel(),
arr[-border:, :].ravel(),
arr[:, :border].ravel(),
arr[:, -border:].ravel(),
])
return int(Counter(border_pixels.tolist()).most_common(1)[0][0])
def _border_mode_rgb(arr: np.ndarray, border: int = 5) -> np.ndarray:
"""Most common RGB colour along the image border."""
h, w, _ = arr.shape
border_pixels = np.concatenate([
arr[:border, :].reshape(-1, 3),
arr[-border:, :].reshape(-1, 3),
arr[:, :border].reshape(-1, 3),
arr[:, -border:].reshape(-1, 3),
], axis=0)
tuples = [tuple(row) for row in border_pixels.tolist()]
most_common = Counter(tuples).most_common(1)[0][0]
return np.array(most_common, dtype=np.uint8)
# ---------------------------------------------------------------------------
# Bounding box from mask
# ---------------------------------------------------------------------------
def _mask_to_crop_box(
disc_mask: np.ndarray,
scale: float,
img_size: Tuple[int, int],
) -> Tuple[int, int, int, int]:
"""
Compute a square crop box centred on the disc with padding = scale * radius.
Returns (left, upper, right, lower) — ready for PIL Image.crop().
Falls back to the full image if no disc pixels are found.
"""
coords = np.argwhere(disc_mask > 0) # (N, 2) in (row, col) order
if coords.size == 0:
w, h = img_size
return (0, 0, w, h)
ys, xs = coords[:, 0], coords[:, 1]
centre_x = float(xs.mean())
centre_y = float(ys.mean())
radius = max(float(xs.max() - xs.min()), float(ys.max() - ys.min())) / 2.0
crop_radius = radius * scale
iw, ih = img_size
left = int(max(0, centre_x - crop_radius))
upper = int(max(0, centre_y - crop_radius))
right = int(min(iw, centre_x + crop_radius))
lower = int(min(ih, centre_y + crop_radius))
# Make square by expanding the shorter side
cw, ch = right - left, lower - upper
if cw < ch:
diff = ch - cw
left = max(0, left - diff // 2)
right = min(iw, right + diff // 2)
elif ch < cw:
diff = cw - ch
upper = max(0, upper - diff // 2)
lower = min(ih, lower + diff // 2)
return (left, upper, right, lower)
scripts/basic_analysis/cnn_logits_rf_cv_v2.py
+399
View File
@@ -0,0 +1,399 @@
#!/usr/bin/env python3
"""v2 port of cnn_logits_rf_cv: CNN logit extraction + RF CV using the v2 PAPILA stack.
Replaces the hardcoded-config v1 version. Data loading, fold splitting, and
feature preparation all go through the v2 stack so that --exclude-cols,
--iop-corr-method, etc. are first-class options.
Usage:
python scripts/basic_analysis/cnn_logits_rf_cv_v2.py \
--eval-mode binary --backbone resnet50 --epochs 40 \
--exclude-cols Phakic/Pseudophakic Axial_Length \
--run-name cnn_rf_nocrop_binary
"""
from __future__ import annotations
import argparse
import random
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import sys
from types import SimpleNamespace
from typing import Dict, List, Tuple
import numpy as np
import pandas as pd
from PIL import Image
import torch
from torch import nn
from torch.utils.data import DataLoader
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, roc_auc_score
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.v2.papila_builders import build_papila_data
from classes.v2.backbones import BACKBONES, load_backbone_weights
from classes.v2.dataset import ClinicalDataset, _ClinicalView
from classes.v2.split_manager import PatientFirstSplitManager
from classes.v2.transforms import build_backbone_transform, build_eval_transform
# ---------------------------------------------------------------------------
# Model
# ---------------------------------------------------------------------------
class CNNHead(nn.Module):
def __init__(self, backbone_name: str, num_classes: int) -> None:
super().__init__()
spec = BACKBONES[backbone_name]
backbone = spec.ctor(weights=spec.weights_default)
if backbone_name.startswith("refuge"):
load_backbone_weights(backbone_name, backbone)
out_dim, backbone = spec.strip(backbone)
self.backbone = backbone
self.head = nn.Linear(out_dim, num_classes)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.head(self.backbone(x))
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _auc_score(y_true: np.ndarray, probs: np.ndarray, num_classes: int) -> float:
try:
if num_classes == 2:
return float(roc_auc_score(y_true, probs[:, 1]))
return float(roc_auc_score(y_true, probs, multi_class="ovr", average="macro"))
except Exception:
return float("nan")
def _train_cnn(
model: nn.Module,
loader: DataLoader,
args,
fold: int,
device: torch.device,
) -> None:
model.train()
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
criterion = nn.CrossEntropyLoss()
for epoch in range(args.epochs):
running_loss = total = correct = 0
for x_img, _meta, y in loader:
x_img = x_img.to(device)
y = y.to(device=device, dtype=torch.long)
optimizer.zero_grad()
logits = model(x_img)
loss = criterion(logits, y)
loss.backward()
optimizer.step()
running_loss += float(loss.item()) * int(y.size(0))
correct += int((logits.argmax(1) == y).sum().item())
total += int(y.size(0))
if (epoch + 1) % args.log_every == 0:
print(
f" [fold {fold+1}] epoch {epoch+1}/{args.epochs} "
f"loss={running_loss/max(total,1):.4f} acc={correct/max(total,1):.4f}",
flush=True,
)
def _infer_logits(
model: nn.Module, loader: DataLoader, device: torch.device
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Returns (y_true, logits, metadata_vectors)."""
model.eval()
y_all, logits_all, md_all = [], [], []
with torch.no_grad():
for x_img, x_md, y in loader:
logits = model(x_img.to(device)).cpu().numpy()
y_np = y.numpy() if torch.is_tensor(y) else np.asarray(y)
md_np = x_md.numpy() if torch.is_tensor(x_md) else np.asarray(x_md)
y_all.append(y_np)
logits_all.append(logits)
md_all.append(md_np)
return (
np.concatenate(y_all),
np.concatenate(logits_all),
np.concatenate(md_all),
)
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def build_parser() -> argparse.ArgumentParser:
ap = argparse.ArgumentParser(
description="CNN logit extraction + RF CV (v2 PAPILA stack)"
)
ap.add_argument("--image-dir", default="Papila/FundusImages")
ap.add_argument("--clinical-dir", default="Papila/ClinicalData")
ap.add_argument("--label-col", default="Diagnosis")
ap.add_argument("--cat-cols", nargs="*", default=["Gender", "Phakic/Pseudophakic"])
ap.add_argument("--exclude-cols", nargs="*", default=[],
help="Feature columns to drop from the clinical feature matrix.")
ap.add_argument("--eval-mode", choices=["binary", "multiclass"], default="binary")
ap.add_argument("--n-splits", type=int, default=5)
ap.add_argument("--fold-seed", type=int, default=42)
ap.add_argument("--holdout-per-class", type=int, default=6,
help="Patients per class reserved for holdout (0 disables).")
ap.add_argument("--holdout-seed", type=int, default=123)
ap.add_argument("--backbone", default="resnet50")
ap.add_argument("--batch-size", type=int, default=8)
ap.add_argument("--epochs", type=int, default=40)
ap.add_argument("--lr", type=float, default=1e-4)
ap.add_argument("--weight-decay", type=float, default=1e-5)
ap.add_argument("--rf-trees", type=int, default=500)
ap.add_argument("--rf-max-depth", type=int, default=None)
ap.add_argument("--rf-min-samples-leaf", type=int, default=1)
ap.add_argument("--num-workers", type=int, default=0)
ap.add_argument("--device", choices=["auto", "cpu", "cuda"], default="auto")
ap.add_argument("--seed", type=int, default=1234)
ap.add_argument("--log-every", type=int, default=5)
ap.add_argument("--run-name", default=None)
ap.add_argument("--output-root", default="analysis_data/basic_analysis/cnn_logits_rf_cv_v2")
ap.add_argument("--iop-corr-method", choices=["ratio", "ols", "lad", "multi"], default="ratio")
ap.add_argument("--iop-drop-raw", action="store_true", default=False)
ap.add_argument("--in-memory-cache", action="store_true", default=True,
help="Cache all images in RAM before training (default: on).")
ap.add_argument("--no-in-memory-cache", action="store_false", dest="in_memory_cache",
help="Disable in-memory image cache.")
ap.add_argument("--cache-workers", type=int, default=4,
help="Threads for prebuilding image cache (default: 4).")
return ap
def _prebuild_image_cache(df: "pd.DataFrame", data, n_workers: int,
resize: int = 256) -> dict:
"""Load, convert to RGB, resize to `resize`px short edge, and cache as uint8 arrays.
Storing pre-resized images means the per-batch transform only has to do
CenterCrop + augmentation + ToTensor + Normalize on a small image rather
than resizing a full-resolution fundus image every step.
"""
paths = list({str(data.get_image_path(row)) for _, row in df.iterrows()})
cache: dict = {}
print(f"[cache] Preloading {len(paths)} images (resize={resize}px) "
f"with {n_workers} threads...", flush=True)
def _load(p: str):
img = Image.open(p)
if img.mode != "RGB":
img = img.convert("RGB")
w, h = img.size
scale = resize / min(w, h)
img = img.resize((round(w * scale), round(h * scale)), Image.BILINEAR)
return p, np.asarray(img, dtype=np.uint8)
with ThreadPoolExecutor(max_workers=max(1, n_workers)) as ex:
for path, arr in ex.map(_load, paths):
cache[path] = arr
ex_shape = cache[paths[0]].shape
print(f"[cache] Done — {len(cache)} images in RAM "
f"({ex_shape[1]}×{ex_shape[0]} each).", flush=True)
return cache
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main() -> None:
args = build_parser().parse_args()
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(args.seed)
if args.device == "auto":
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
else:
device = torch.device(args.device)
print(f"Device: {device}", flush=True)
run_name = args.run_name or time.strftime("%Y%m%d_%H%M%S")
out_dir = Path(args.output_root) / run_name
out_dir.mkdir(parents=True, exist_ok=True)
# ---------- data ----------
print("Loading PAPILA data...", flush=True)
data = build_papila_data(
image_dir=args.image_dir,
clinical_dir=args.clinical_dir,
label_col=args.label_col,
cat_cols=list(args.cat_cols),
n_splits=args.n_splits,
random_seed=args.fold_seed,
iop_corr_method=args.iop_corr_method,
iop_drop_raw=args.iop_drop_raw,
exclude_cols=list(args.exclude_cols or []),
)
print(f"Loaded: {len(data.df)} rows feature_dim={data.feature_dim}", flush=True)
df_mode = data.df.copy()
if args.eval_mode == "binary":
df_mode = df_mode[df_mode[args.label_col].isin([0, 1])].reset_index(drop=True)
num_classes = 2 if args.eval_mode == "binary" else int(df_mode[args.label_col].nunique())
print(
f"eval_mode={args.eval_mode} num_classes={num_classes} "
f"rows={len(df_mode)} patients={df_mode['Patient ID'].nunique()}",
flush=True,
)
# ---------- splits ----------
split_manager = PatientFirstSplitManager(
patient_col="Patient ID", label_col=args.label_col
)
split_args = SimpleNamespace(
eval_mode=args.eval_mode,
holdout_per_class=args.holdout_per_class,
holdout_seed=args.holdout_seed,
n_splits=args.n_splits,
fold_seed=args.fold_seed,
)
clinical_ns = SimpleNamespace(df=df_mode, label_col=args.label_col)
plans = split_manager.build_plans(clinical=clinical_ns, args=split_args, profile=None)
n_folds = min(args.n_splits, len(plans))
if plans and plans[0].holdout is not None:
plans[0].holdout.to_csv(out_dir / "holdout_patients.csv", index=False)
# ---------- image cache ----------
image_cache = None
if args.in_memory_cache:
image_cache = _prebuild_image_cache(df_mode, data, args.cache_workers)
# ---------- transforms ----------
train_tf = build_backbone_transform(args.backbone, augment=True)
eval_tf = build_eval_transform(args.backbone)
rows: List[Dict] = []
holdout_rows: List[Dict] = []
for fold, split in enumerate(plans[:n_folds]):
print(f"\n[info] Fold {fold+1}/{n_folds}", flush=True)
random.seed(args.seed + fold * 100)
np.random.seed(args.seed + fold * 100)
torch.manual_seed(args.seed + fold * 100)
view_train = _ClinicalView(data, split.train)
view_val = _ClinicalView(data, split.val)
dl_train = DataLoader(
ClinicalDataset(view_train, train_tf, image_cache=image_cache),
batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers,
)
dl_val = DataLoader(
ClinicalDataset(view_val, eval_tf, image_cache=image_cache),
batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers,
)
dl_holdout = None
if split.holdout is not None and not split.holdout.empty:
dl_holdout = DataLoader(
ClinicalDataset(_ClinicalView(data, split.holdout), eval_tf,
image_cache=image_cache),
batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers,
)
# Train CNN
model = CNNHead(args.backbone, num_classes=num_classes).to(device)
_train_cnn(model, dl_train, args, fold=fold, device=device)
# Extract logits (re-run train without augmentation for RF features)
dl_train_eval = DataLoader(
ClinicalDataset(view_train, eval_tf, image_cache=image_cache),
batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers,
)
y_tr, log_tr, md_tr = _infer_logits(model, dl_train_eval, device)
y_va, log_va, md_va = _infer_logits(model, dl_val, device)
np.save(out_dir / f"fold{fold}_train_logits.npy", log_tr)
np.save(out_dir / f"fold{fold}_val_logits.npy", log_va)
X_tr = np.concatenate([log_tr, md_tr], axis=1)
X_va = np.concatenate([log_va, md_va], axis=1)
rf = RandomForestClassifier(
n_estimators=args.rf_trees,
max_depth=args.rf_max_depth,
min_samples_leaf=args.rf_min_samples_leaf,
class_weight="balanced",
random_state=args.fold_seed + fold,
n_jobs=-1,
)
rf.fit(X_tr, y_tr)
p_va = rf.predict_proba(X_va)
pred_va = np.argmax(p_va, axis=1)
rows.append({
"fold": fold,
"val_acc": float(accuracy_score(y_va, pred_va)),
"val_auc": _auc_score(y_va, p_va, num_classes),
"n_val": int(len(y_va)),
})
if dl_holdout is not None:
y_ho, log_ho, md_ho = _infer_logits(model, dl_holdout, device)
np.save(out_dir / f"fold{fold}_holdout_logits.npy", log_ho)
X_ho = np.concatenate([log_ho, md_ho], axis=1)
p_ho = rf.predict_proba(X_ho)
pred_ho = np.argmax(p_ho, axis=1)
holdout_rows.append({
"fold": fold,
"holdout_acc": float(accuracy_score(y_ho, pred_ho)),
"holdout_auc": _auc_score(y_ho, p_ho, num_classes),
"n_holdout": int(len(y_ho)),
})
print(
f"[info] Fold {fold+1} RF: val_acc={rows[-1]['val_acc']:.4f}"
f" val_auc={rows[-1]['val_auc']:.4f}"
f" | holdout_acc={holdout_rows[-1]['holdout_acc']:.4f}"
f" holdout_auc={holdout_rows[-1]['holdout_auc']:.4f}",
flush=True,
)
else:
print(
f"[info] Fold {fold+1} RF: val_acc={rows[-1]['val_acc']:.4f}"
f" val_auc={rows[-1]['val_auc']:.4f}",
flush=True,
)
# ---------- save + summarise ----------
fold_df = pd.DataFrame(rows)
fold_df.to_csv(out_dir / "rf_val_metrics.csv", index=False)
print("\nRF validation metrics:")
print(fold_df.to_string(index=False, float_format=lambda x: f"{x:.4f}"))
if holdout_rows:
ho_df = pd.DataFrame(holdout_rows)
ho_df.to_csv(out_dir / "rf_holdout_metrics.csv", index=False)
print("\nRF holdout metrics:")
print(ho_df.to_string(index=False, float_format=lambda x: f"{x:.4f}"))
print(
f"\nMeans: val_acc={fold_df['val_acc'].mean():.4f}"
f" val_auc={fold_df['val_auc'].mean():.4f}"
f" holdout_acc={ho_df['holdout_acc'].mean():.4f}"
f" holdout_auc={ho_df['holdout_auc'].mean():.4f}"
)
else:
print(
f"\nMeans: val_acc={fold_df['val_acc'].mean():.4f}"
f" val_auc={fold_df['val_auc'].mean():.4f}"
)
print(f"\nSaved outputs to: {out_dir}", flush=True)
if __name__ == "__main__":
main()
scripts/basic_analysis/compare_dual_eye_towers.py → scripts/exploratory/compare_dual_eye_towers.py
View File
scripts/basic_analysis/compare_siamese_tower.py → scripts/exploratory/compare_siamese_tower.py
View File
scripts/basic_analysis/compare_siamese_v2.py → scripts/exploratory/compare_siamese_v2.py
View File
scripts/grid_search_analytics/best_holdout_multiclass.py → scripts/exploratory/grid_search_analytics/best_holdout_multiclass.py
View File
scripts/grid_search_analytics/check_crop_cache_vs_gt.py → scripts/exploratory/grid_search_analytics/check_crop_cache_vs_gt.py
View File
scripts/grid_search_analytics/compare_suspect_geometry_vs_image.py → scripts/exploratory/grid_search_analytics/compare_suspect_geometry_vs_image.py
View File
scripts/grid_search_analytics/derived_analysis.py → scripts/exploratory/grid_search_analytics/derived_analysis.py
View File
scripts/grid_search_analytics/derived_statistics.py → scripts/exploratory/grid_search_analytics/derived_statistics.py
View File
scripts/grid_search_analytics/grid_search_heatmap.py → scripts/exploratory/grid_search_analytics/grid_search_heatmap.py
View File
scripts/grid_search_analytics/holdout_roc_for_run.py → scripts/exploratory/grid_search_analytics/holdout_roc_for_run.py
View File
scripts/grid_search_analytics/param_perf_correlations.py → scripts/exploratory/grid_search_analytics/param_perf_correlations.py
View File
scripts/grid_search_analytics/rerun_grid_item.py → scripts/exploratory/grid_search_analytics/rerun_grid_item.py
View File
scripts/grid_search_analytics/rerun_grid_item_v2.py → scripts/exploratory/grid_search_analytics/rerun_grid_item_v2.py
View File
scripts/grid_search_analytics/run_derived_analysis.py → scripts/exploratory/grid_search_analytics/run_derived_analysis.py
View File
scripts/exploratory/iop_correction_analysis.py
+269
View File
@@ -0,0 +1,269 @@
#!/usr/bin/env python3
"""
Compare Perkins→Pneumatic IOP conversion methods:
- Current: fixed ratio (1.158)
- OLS linear regression (minimises MSE)
- LAD linear regression (minimises MAE — robust to outliers)
- Multiple regression: Perkins + Pachymetry (OLS)
Also reports on the impact of dropping IOP_raw.
Run from repo root:
python scripts/exploratory/iop_correction_analysis.py
"""
from __future__ import annotations
import sys
from pathlib import Path
import numpy as np
import pandas as pd
from scipy import stats
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
REPO = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO))
CURRENT_RATIO = 1.158
# ── load raw clinical data ────────────────────────────────────────────────────
def load_raw() -> pd.DataFrame:
dfs = []
for fname in ("patient_data_od.xlsx", "patient_data_os.xlsx"):
df = pd.read_excel(REPO / "Papila/ClinicalData" / fname, header=1)
eye = "OD" if "od" in fname else "OS"
df["eyeID"] = eye
dfs.append(df)
return pd.concat(dfs, ignore_index=True)
df = load_raw()
print(f"Total rows: {len(df)}")
print(f"Columns with IOP: {[c for c in df.columns if 'iop' in c.lower() or c in ('Pneumatic','Perkins')]}")
# ── find paired rows ──────────────────────────────────────────────────────────
paired = df.dropna(subset=["Pneumatic", "Perkins", "Pachymetry"]).copy()
pneumatic = paired["Pneumatic"].values.astype(float)
perkins = paired["Perkins"].values.astype(float)
pachymetry = paired["Pachymetry"].values.astype(float)
print(f"\nPaired obs (Pneumatic + Perkins + Pachymetry): n={len(paired)}")
print(f" Pneumatic: mean={pneumatic.mean():.2f} std={pneumatic.std():.2f} "
f"range=[{pneumatic.min():.1f}, {pneumatic.max():.1f}]")
print(f" Perkins: mean={perkins.mean():.2f} std={perkins.std():.2f} "
f"range=[{perkins.min():.1f}, {perkins.max():.1f}]")
# ── current ratio ─────────────────────────────────────────────────────────────
ratio_obs = pneumatic / perkins
ratio_mean = ratio_obs.mean()
ratio_pred = perkins * CURRENT_RATIO
ratio_resid = pneumatic - ratio_pred
ratio_mae = np.abs(ratio_resid).mean()
ratio_rmse = np.sqrt((ratio_resid ** 2).mean())
print(f"\n── Current ratio approach ────────────────────────────────")
print(f" Observed Pneumatic/Perkins ratio: mean={ratio_mean:.4f} "
f"std={ratio_obs.std():.4f} range=[{ratio_obs.min():.3f}, {ratio_obs.max():.3f}]")
print(f" Hardcoded ratio used: {CURRENT_RATIO}")
print(f" MAE: {ratio_mae:.3f} mmHg")
print(f" RMSE: {ratio_rmse:.3f} mmHg")
# ── OLS regression ────────────────────────────────────────────────────────────
slope, intercept, r, p, se = stats.linregress(perkins, pneumatic)
reg_pred = slope * perkins + intercept
reg_resid = pneumatic - reg_pred
reg_mae = np.abs(reg_resid).mean()
reg_rmse = np.sqrt((reg_resid ** 2).mean())
print(f"\n── OLS regression (minimises MSE): Pneumatic = slope * Perkins + intercept ──")
print(f" slope={slope:.4f} intercept={intercept:.4f}")
print(f" R²={r**2:.4f} p={p:.4e}")
print(f" MAE: {reg_mae:.3f} mmHg")
print(f" RMSE: {reg_rmse:.3f} mmHg")
print(f" Improvement over ratio — MAE: {ratio_mae - reg_mae:+.3f} RMSE: {ratio_rmse - reg_rmse:+.3f}")
# LAD regression (minimises MAE) — more robust to outliers
from scipy.optimize import minimize
def lad_loss(params):
a, b = params
return np.abs(pneumatic - (a * perkins + b)).mean()
lad_res = minimize(lad_loss, x0=[slope, intercept], method="Nelder-Mead")
lad_slope, lad_intercept = lad_res.x
lad_pred = lad_slope * perkins + lad_intercept
lad_resid = pneumatic - lad_pred
lad_mae = np.abs(lad_resid).mean()
lad_rmse = np.sqrt((lad_resid ** 2).mean())
print(f"\n── LAD regression (minimises MAE — robust to outliers) ──────")
print(f" slope={lad_slope:.4f} intercept={lad_intercept:.4f}")
print(f" MAE: {lad_mae:.3f} mmHg")
print(f" RMSE: {lad_rmse:.3f} mmHg")
print(f" Improvement over ratio — MAE: {ratio_mae - lad_mae:+.3f} RMSE: {ratio_rmse - lad_rmse:+.3f}")
# ── Multiple regression: Perkins + Pachymetry ─────────────────────────────────
# Pachymetry affects Perkins (applanation) more than Pneumatic, so including
# CCT should absorb some instrument-specific bias.
# Design matrix: [Perkins, Pachymetry, 1]
from numpy.linalg import lstsq
X_multi = np.column_stack([perkins, pachymetry, np.ones(len(perkins))])
coeffs, _, _, _ = lstsq(X_multi, pneumatic, rcond=None)
coef_perkins, coef_pachy, coef_intercept = coeffs
multi_pred = X_multi @ coeffs
multi_resid = pneumatic - multi_pred
multi_mae = np.abs(multi_resid).mean()
multi_rmse = np.sqrt((multi_resid ** 2).mean())
# R² for the multiple model
ss_res = (multi_resid ** 2).sum()
ss_tot = ((pneumatic - pneumatic.mean()) ** 2).sum()
multi_r2 = 1 - ss_res / ss_tot
# Partial correlation of Pachymetry with residual after removing Perkins effect
perkins_resid = pneumatic - (slope * perkins + intercept)
pachy_r, pachy_p = stats.pearsonr(pachymetry, perkins_resid)
print(f"\n── Multiple OLS (Perkins + Pachymetry, n={len(paired)}) ──────────────")
print(f" Pneumatic = {coef_perkins:.4f}×Perkins + {coef_pachy:.5f}×Pachymetry + {coef_intercept:.4f}")
print(f" R²={multi_r2:.4f} (vs simple OLS R²={r**2:.4f})")
print(f" Pachymetry partial corr with OLS residuals: r={pachy_r:.3f} p={pachy_p:.4f}")
print(f" MAE: {multi_mae:.3f} mmHg (vs ratio {ratio_mae:.3f})")
print(f" RMSE: {multi_rmse:.3f} mmHg (vs ratio {ratio_rmse:.3f})")
print(f" Improvement over ratio — MAE: {ratio_mae - multi_mae:+.3f} RMSE: {ratio_rmse - multi_rmse:+.3f}")
print(f" NOTE: n={len(paired)} with 3 parameters — interpret with caution.")
# How much does the intercept matter at typical IOP values?
typical_iop = np.array([10, 15, 20, 25])
print(f"\n Comparison at typical Perkins values:")
print(f" {'Perkins':>8} {'Ratio':>10} {'OLS':>10} {'LAD':>10}")
for v in typical_iop:
ratio_v = v * CURRENT_RATIO
ols_v = slope * v + intercept
lad_v = lad_slope * v + lad_intercept
print(f" {v:>8.1f} {ratio_v:>10.2f} {ols_v:>10.2f} {lad_v:>10.2f}")
# ── IOP_raw coverage ──────────────────────────────────────────────────────────
pneumatic_only = df["Pneumatic"].notna() & df["Perkins"].isna()
perkins_only = df["Pneumatic"].isna() & df["Perkins"].notna()
both = df["Pneumatic"].notna() & df["Perkins"].notna()
neither = df["Pneumatic"].isna() & df["Perkins"].isna()
print(f"\n── IOP measurement coverage ──────────────────────────────")
print(f" Pneumatic only: {pneumatic_only.sum()}")
print(f" Perkins only: {perkins_only.sum()}")
print(f" Both: {both.sum()}")
print(f" Neither: {neither.sum()}")
print(f" Rows where IOP_raw == IOP_corr (no pachy correction): ", end="")
# ── plot ──────────────────────────────────────────────────────────────────────
fig = plt.figure(figsize=(16, 5), layout="constrained")
gs = gridspec.GridSpec(1, 3, figure=fig)
x_line = np.linspace(perkins.min() - 1, perkins.max() + 1, 100)
# ── Row 1: conversion fits ────────────────────────────────────────────────────
# 1a. Scatter with all four fits
ax1 = fig.add_subplot(gs[0, :2]) # spans first two columns
ax1.scatter(perkins, pneumatic, alpha=0.6, s=35, color="gray", zorder=3, label="Observed pairs (n=41)")
ax1.plot(x_line, x_line * CURRENT_RATIO, "r--", lw=2, label=f"Ratio ×{CURRENT_RATIO} MAE={ratio_mae:.2f}")
ax1.plot(x_line, slope * x_line + intercept, "b-", lw=2, label=f"OLS (slope={slope:.3f}, int={intercept:.2f}) MAE={reg_mae:.2f}")
ax1.plot(x_line, lad_slope * x_line + lad_intercept, "g-", lw=2, label=f"LAD (slope={lad_slope:.3f}, int={lad_intercept:.2f}) MAE={lad_mae:.2f}")
# Multi-reg projected at mean CCT
pachy_mean, pachy_sd = pachymetry.mean(), pachymetry.std()
multi_mean_line = coef_perkins * x_line + coef_pachy * pachy_mean + coef_intercept
ax1.plot(x_line, multi_mean_line, color="purple", lw=2, ls="-.",
label=f"Multi (Perkins+CCT) @ mean CCT MAE={multi_mae:.2f}")
ax1.set_xlabel("Perkins IOP (mmHg)")
ax1.set_ylabel("Pneumatic IOP (mmHg)")
ax1.set_title("Perkins → Pneumatic conversion: all methods")
ax1.legend(fontsize=8)
# 1b. MAE / RMSE bar chart
ax_bar = fig.add_subplot(gs[0, 2]) # third column
bar_methods = ["Ratio", "OLS", "LAD", "Multi\n(+CCT)"]
maes = [ratio_mae, reg_mae, lad_mae, multi_mae]
rmses = [ratio_rmse, reg_rmse, lad_rmse, multi_rmse]
bar_colors = ["#e05c5c", "#5c7de0", "#5cc97c", "#9b59b6"]
bx = np.arange(4)
w = 0.35
ax_bar.bar(bx - w/2, maes, w, label="MAE", color=bar_colors)
ax_bar.bar(bx + w/2, rmses, w, label="RMSE", color=bar_colors, alpha=0.5)
ax_bar.set_xticks(bx); ax_bar.set_xticklabels(bar_methods, fontsize=8)
ax_bar.set_ylabel("Error (mmHg)")
ax_bar.set_title("MAE & RMSE comparison")
ax_bar.legend(fontsize=9)
ax_bar.set_ylim(0, max(rmses) * 1.25)
for i, (mae, rmse) in enumerate(zip(maes, rmses)):
ax_bar.text(i - w/2, mae + 0.05, f"{mae:.2f}", ha="center", va="bottom", fontsize=8)
ax_bar.text(i + w/2, rmse + 0.05, f"{rmse:.2f}", ha="center", va="bottom", fontsize=8)
out = REPO / "analysis_data/iop_correction_comparison.png"
fig.savefig(out, dpi=150)
print(f"\nPlot saved to {out}")
# ── Figure 2: Pachymetry analysis ─────────────────────────────────────────────
fig2, axes2 = plt.subplots(1, 3, figsize=(15, 5))
pachy_norm = (pachymetry - pachymetry.mean()) / pachymetry.std()
# 2a. Pachymetry vs Pneumatic–Perkins difference
ax = axes2[0]
diff = pneumatic - perkins
m, b, rr, pp, _ = stats.linregress(pachymetry, diff)
ax.scatter(pachymetry, diff, alpha=0.6, s=35, color="steelblue")
px = np.linspace(pachymetry.min() - 5, pachymetry.max() + 5, 100)
ax.plot(px, m * px + b, "r-", lw=2, label=f"r={rr:.2f} p={pp:.3f}")
ax.axhline(0, color="k", lw=0.7, ls="--")
ax.set_xlabel("Pachymetry (μm)")
ax.set_ylabel("Pneumatic − Perkins (mmHg)")
ax.set_title("Does CCT predict the instrument gap?")
ax.legend(fontsize=9)
# 2b. Residuals from ratio vs Pachymetry (coloured by size)
ax = axes2[1]
sc = ax.scatter(pachymetry, ratio_resid, c=perkins, cmap="viridis", alpha=0.7, s=35)
plt.colorbar(sc, ax=ax, label="Perkins IOP")
m2, b2, rr2, pp2, _ = stats.linregress(pachymetry, ratio_resid)
ax.plot(px, m2 * px + b2, "r-", lw=2, label=f"r={rr2:.2f} p={pp2:.3f}")
ax.axhline(0, color="k", lw=0.7, ls="--")
ax.set_xlabel("Pachymetry (μm)")
ax.set_ylabel("Ratio residual (mmHg)")
ax.set_title("Ratio residuals vs Pachymetry\n(colour = Perkins IOP)")
ax.legend(fontsize=9)
# 2c. Multiple regression residuals vs Pachymetry (should be flat if absorbed)
ax = axes2[2]
m3, b3, rr3, pp3, _ = stats.linregress(pachymetry, multi_resid)
ax.scatter(pachymetry, multi_resid, alpha=0.6, s=35, color="darkorange")
ax.plot(px, m3 * px + b3, "r-", lw=2, label=f"r={rr3:.2f} p={pp3:.3f}")
ax.axhline(0, color="k", lw=0.7, ls="--")
ax.set_xlabel("Pachymetry (μm)")
ax.set_ylabel("Multi-reg residual (mmHg)")
ax.set_title("Multi-reg residuals vs Pachymetry\n(flat = Pachymetry effect absorbed)")
ax.legend(fontsize=9)
# shared y-axis
ylim2 = max(np.abs(diff).max(), np.abs(ratio_resid).max(), np.abs(multi_resid).max()) + 1
for ax in axes2[1:]:
ax.set_ylim(-ylim2, ylim2)
fig2.suptitle(
f"Pachymetry as a covariate (n={len(paired)}, CCT mean={pachymetry.mean():.0f}±{pachymetry.std():.0f} μm)",
fontsize=11,
)
fig2.tight_layout()
out2 = REPO / "analysis_data/iop_pachymetry_analysis.png"
fig2.savefig(out2, dpi=150)
print(f"Plot saved to {out2}")
scripts/legacy/classic_ml_roc.py
-326
View File
@@ -1,326 +0,0 @@
#!/usr/bin/env python3
"""Run classical ML models and plot *combined* ROC curves (multimodel overlays).
Keeps your original workflow for folds/tests exactly the same.
Only changes: collects predictions per test and makes:
• One ROC plot per class (OvR), overlaying all models
• One binary ROC plot (Healthy vs Glaucoma), overlaying all models
"""
import os
import re
from pathlib import Path
from typing import Iterable, List, Tuple, Dict, Optional
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler, label_binarize
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.metrics import roc_curve, auc
from classes import build_papila_clinical
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
SPLIT_ROOT = Path("HelpCode/kfold")
TRUST_INDEX_COL = False
# ---------------------------------------------------------------------------
# Feature matrix
# ---------------------------------------------------------------------------
def build_feature_matrix(clinical):
df = clinical.df.copy()
scalars = ["Age", "dioptre_1", "dioptre_2", "astigmatism", "Pachymetry", "Axial_Length", "IOP_corr"]
cats = ["Gender", "Phakic/Pseudophakic"]
X = pd.concat([df[scalars], pd.get_dummies(df[cats].astype("category"), drop_first=False, prefix=cats)], axis=1)
y = df[clinical.label_col].astype(int).values
return X, y, scalars, df # X keeps NaNs; we impute per-fold
# ---------------------------------------------------------------------------
# Models with tuned hyper-parameters (unchanged)
# ---------------------------------------------------------------------------
def make_models() -> Dict[str, Pipeline]:
return {
"LogReg": Pipeline([
("scaler", StandardScaler()),
("clf", LogisticRegression(
C=1,
class_weight="balanced",
max_iter=200,
solver="lbfgs",
multi_class="auto")),
]),
"kNN": Pipeline([
("scaler", StandardScaler()),
("clf", KNeighborsClassifier(
n_neighbors=11, weights="distance")),
]),
"RF": Pipeline([
("clf", RandomForestClassifier(n_estimators=200, max_depth=8,
min_samples_split=4, random_state=42)),
]),
"SVM": Pipeline([
("scaler", StandardScaler()),
("clf", SVC(C=10, kernel="rbf", gamma=0.1, probability=True)),
]),
}
# ---------------------------------------------------------------------------
# Split helpers copied from paper_clinical_baselines_official.py (unchanged)
# ---------------------------------------------------------------------------
_FNAME_RE = re.compile(r"RET\s*(\d+)\s*([Oo][DSs])\.jpg$", re.IGNORECASE)
def _read_sheet_any(p: Path) -> pd.DataFrame:
if p.suffix.lower() == ".xlsx":
return pd.read_excel(p)
if p.suffix.lower() == ".csv":
return pd.read_csv(p)
if p.suffix.lower() == ".txt":
lines = [ln.strip() for ln in p.read_text(encoding="utf-8", errors="ignore").splitlines() if ln.strip()]
return pd.DataFrame({"filename": lines})
raise ValueError(f"Unsupported split file type: {p.suffix}")
def _normcols(cols: List[str]) -> Dict[str, str]:
def norm(s: str) -> str:
return re.sub(r"[^a-z0-9]", "", s.lower())
return {norm(c): c for c in cols}
def _parse_fname_to_pid_eye(fname: str) -> Optional[Tuple[int, str]]:
base = os.path.basename(str(fname))
m = _FNAME_RE.search(base.replace(" ", ""))
if not m:
return None
return int(m.group(1)), m.group(2).upper()
def _rows_from_sheet(sheet: pd.DataFrame, df_master: pd.DataFrame) -> List[int]:
cols = _normcols(list(sheet.columns))
if "filename" in cols:
fn_col = cols["filename"]
lookup: Dict[str, List[int]] = {}
for i, (pid, eye) in enumerate(zip(df_master["Patient ID"].astype(int), df_master["eyeID"].astype(str))):
lookup.setdefault(f"{pid}|{eye.upper()}", []).append(i)
rows: List[int] = []
for fn in sheet[fn_col].astype(str).tolist():
pe = _parse_fname_to_pid_eye(fn)
if pe is None:
continue
pid, eye = pe
rows.extend(lookup.get(f"{pid}|{eye}", []))
return rows
if "patientid" in cols and "eyeid" in cols:
pid_col, eye_col = cols["patientid"], cols["eyeid"]
lookup = {}
for i, (pid, eye) in enumerate(zip(df_master["Patient ID"].astype(int), df_master["eyeID"].astype(str))):
lookup.setdefault(f"{pid}|{eye.upper()}", []).append(i)
rows = []
for pid, eye in zip(sheet[pid_col], sheet[eye_col]):
rows.extend(lookup.get(f"{int(pid)}|{str(eye).upper()}", []))
return rows
if TRUST_INDEX_COL and "index" in cols:
idx = sheet[cols["index"]].astype(int).tolist()
n = len(df_master)
return [i for i in idx if 0 <= i < n]
raise RuntimeError("Split sheet missing usable columns")
def _pair_train_test_files(dir_train: Path, dir_test: Path) -> List[Tuple[Path, Path]]:
def fold_key(p: Path) -> str:
m = re.search(r"(\d+)", p.stem)
return m.group(1) if m else p.stem.lower()
trains = sorted([p for p in dir_train.iterdir() if p.is_file() and p.suffix.lower() in (".xlsx", ".csv", ".txt")], key=fold_key)
tests = sorted([p for p in dir_test.iterdir() if p.is_file() and p.suffix.lower() in (".xlsx", ".csv", ".txt")], key=fold_key)
return [(trains[i], tests[i]) for i in range(min(len(trains), len(tests)))]
def iter_official_folds_xlsx(clinical, split_root: Path, test_name: str) -> Iterable[Tuple[pd.DataFrame, pd.DataFrame]]:
df_master = clinical.df.copy()
test_dir = split_root / test_name
dir_train = test_dir / "Train"
dir_test = test_dir / "Test"
if not dir_train.exists() or not dir_test.exists():
raise FileNotFoundError(f"Expected: {dir_train} and {dir_test}")
for train_file, test_file in _pair_train_test_files(dir_train, dir_test):
sh_tr, sh_te = _read_sheet_any(train_file), _read_sheet_any(test_file)
tr_rows, te_rows = _rows_from_sheet(sh_tr, df_master), _rows_from_sheet(sh_te, df_master)
tr_df, te_df = df_master.iloc[tr_rows].copy(), df_master.iloc[te_rows].copy()
yield tr_df, te_df
# ---------------------------------------------------------------------------
# Utilities (unchanged)
# ---------------------------------------------------------------------------
def _prepare_fold_X(X: pd.DataFrame, scalars: List[str], tr_idx: np.ndarray, te_idx: np.ndarray):
Xtr, Xte = X.iloc[tr_idx].copy(), X.iloc[te_idx].copy()
med = Xtr[scalars].median(numeric_only=True)
Xtr[scalars] = Xtr[scalars].fillna(med)
Xte[scalars] = Xte[scalars].fillna(med)
return Xtr.values.astype(np.float32), Xte.values.astype(np.float32)
# ---------------------------------------------------------------------------
# NEW: combined plotting helpers (multimodel overlays)
# ---------------------------------------------------------------------------
def _plot_multiclass_overlay(y_true: np.ndarray, prob_dict: Dict[str, np.ndarray], out_dir: Path, test_tag: str):
"""One figure per class (OvR), overlaying all models."""
n_classes = next(iter(prob_dict.values())).shape[1]
class_names = [f"Class{k}" for k in range(n_classes)]
y_bin = label_binarize(y_true, classes=list(range(n_classes)))
for k in range(n_classes):
fig, ax = plt.subplots(figsize=(6, 5))
for model_name, proba in prob_dict.items():
fpr, tpr, _ = roc_curve(y_bin[:, k], proba[:, k])
auc_val = auc(fpr, tpr)
ax.plot(fpr, tpr, lw=1.8, label=f"{model_name} (AUC={auc_val:.3f})")
ax.plot([0, 1], [0, 1], "k--", lw=1)
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(f"{class_names[k]} vs Rest — {test_tag}")
ax.legend(loc="lower right")
ax.grid(True, alpha=0.3, linestyle="--")
fig.tight_layout()
fig.savefig(out_dir / f"{test_tag}_{class_names[k]}.png", dpi=170)
plt.close(fig)
def _plot_binary_overlay(y_true: np.ndarray, prob1d_dict: Dict[str, np.ndarray], out_dir: Path, test_tag: str):
"""One figure (Healthy vs Glaucoma), overlaying all models. Assumes y_true ∈ {0,1}."""
fig, ax = plt.subplots(figsize=(6, 5))
any_curve = False
for model_name, scores in prob1d_dict.items():
if scores.size == 0:
continue
fpr, tpr, _ = roc_curve(y_true, scores, pos_label=1)
auc_val = auc(fpr, tpr)
ax.plot(fpr, tpr, lw=1.8, label=f"{model_name} (AUC={auc_val:.3f})")
any_curve = True
ax.plot([0, 1], [0, 1], "k--", lw=1)
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(f"Binary Healthy vs Glaucoma — {test_tag}")
if any_curve:
ax.legend(loc="lower right")
ax.grid(True, alpha=0.3, linestyle="--")
fig.tight_layout()
fig.savefig(out_dir / f"{test_tag}_binary.png", dpi=170)
plt.close(fig)
# ---------------------------------------------------------------------------
# Main (same folds/tests flow; only result collation & plotting changed)
# ---------------------------------------------------------------------------
def main():
clinical = build_papila_clinical(
image_dir="Papila/FundusImages",
clinical_dir="Papila/ClinicalData",
label_col="Diagnosis",
cat_cols=["Gender", "Phakic/Pseudophakic"],
)
X, y, scalars, _ = build_feature_matrix(clinical)
models = make_models()
out_dir = Path("analysis_data/roc_baselines")
out_dir.mkdir(parents=True, exist_ok=True)
tests = [
("Test 3", False), ("Test 4", True)
] if (SPLIT_ROOT / "Test 3").exists() else [
("Test 1", False), ("Test 2", True)
]
for test_name, is_binary in tests:
# Collect per-model probabilities following your original per-model loop.
# For multiclass: dict[model] -> (N, C)
# For binary: dict[model] -> (N,) (probability of class 1)
prob_dict_multi: Dict[str, np.ndarray] = {}
prob_dict_bin: Dict[str, np.ndarray] = {}
y_ref_multi: Optional[np.ndarray] = None
y_ref_bin: Optional[np.ndarray] = None
for model_name, model in models.items():
y_all: List[np.ndarray] = []
p_all: List[np.ndarray] = []
for fold_idx, (train_df, test_df) in enumerate(iter_official_folds_xlsx(clinical, SPLIT_ROOT, test_name), 1):
# Keep your exact masking/handling
dup_rows = set(train_df.index).intersection(set(test_df.index))
shared_pids = set(train_df["Patient ID"]).intersection(set(test_df["Patient ID"]))
if test_name in ("Test 1", "Test 2") and shared_pids:
train_df = train_df[~train_df["Patient ID"].isin(shared_pids)].copy()
dup_rows = set(train_df.index).intersection(set(test_df.index))
shared_pids = set(train_df["Patient ID"]).intersection(set(test_df["Patient ID"]))
tr_idx, te_idx = train_df.index.values, test_df.index.values
if is_binary:
# original binary handling: drop Suspects on both sets
mask_tr = np.isin(y[tr_idx], [0, 1])
mask_te = np.isin(y[te_idx], [0, 1])
if not mask_tr.any() or not mask_te.any():
# skip empty fold (keeps behavior safe without changing fold logic)
continue
Xtr, Xte = _prepare_fold_X(X, scalars, tr_idx[mask_tr], te_idx[mask_te])
ytr, yte = y[tr_idx][mask_tr], y[te_idx][mask_te]
else:
Xtr, Xte = _prepare_fold_X(X, scalars, tr_idx, te_idx)
ytr, yte = y[tr_idx], y[te_idx]
# Fit and score (unchanged approach)
model.fit(Xtr, ytr)
if is_binary:
if hasattr(model[-1], "predict_proba"):
prob = model.predict_proba(Xte)[:, 1]
else:
dec = model.decision_function(Xte)
prob = 1.0 / (1.0 + np.exp(-dec)) if np.ptp(dec) > 0 else np.full_like(dec, 0.5)
y_all.append(yte)
p_all.append(prob)
else:
if hasattr(model[-1], "predict_proba"):
prob = model.predict_proba(Xte)
else:
dec = model.decision_function(Xte)
if dec.ndim == 1:
dec = np.stack([-dec, dec], axis=1)
e = np.exp(dec - dec.max(axis=1, keepdims=True))
prob = e / e.sum(axis=1, keepdims=True)
y_all.append(yte)
p_all.append(prob)
if not y_all:
# No valid folds for this model under this test (e.g., all-bad after mask); skip
continue
y_cat = np.concatenate(y_all)
p_cat = np.concatenate(p_all)
if is_binary:
# Store 1D scores per model
prob_dict_bin[model_name] = p_cat
if y_ref_bin is None:
y_ref_bin = y_cat
else:
# Align lengths defensively (should match in normal use)
n = min(len(y_ref_bin), len(y_cat))
y_ref_bin = y_ref_bin[:n]
prob_dict_bin[model_name] = prob_dict_bin[model_name][:n]
else:
# Store (N, C) per model
prob_dict_multi[model_name] = p_cat
if y_ref_multi is None:
y_ref_multi = y_cat
else:
# Align lengths defensively (should match in normal use)
n = min(len(y_ref_multi), len(y_cat))
y_ref_multi = y_ref_multi[:n]
prob_dict_multi[model_name] = prob_dict_multi[model_name][:n, :]
tag = test_name.replace(" ", "")
# Produce overlays
if prob_dict_multi and y_ref_multi is not None:
_plot_multiclass_overlay(y_ref_multi, prob_dict_multi, out_dir, tag)
if prob_dict_bin and y_ref_bin is not None:
_plot_binary_overlay(y_ref_bin, prob_dict_bin, out_dir, tag)
if __name__ == "__main__":
main()
scripts/legacy/eval_papila_classifier.py
-384
View File
@@ -1,384 +0,0 @@
"""Evaluate REFUGE-trained classifier on Papila images using UNet crops."""
from __future__ import annotations
import argparse
import csv
from pathlib import Path
from typing import Dict, List, Optional, Sequence, Set
import numpy as np
import torch
from torch.utils.data import DataLoader
from tqdm import tqdm
from PIL import Image, ImageDraw
import sys
REPO_ROOT = Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.refuge_preprocessing import RefugePreprocessing, RefugeSample
from classes.refuge_segmentation import RefugeSegmentation
from classes.refuge_classification import (
RefugeClassification,
RefugeClassificationDataset,
RefugeClassificationRecord,
UNetGeometryProvider,
_default_image_transform,
_geometry_from_mask,
)
from classes.backbones import BACKBONES, load_backbone_weights
from classes.unet_segmenter import UNetSegmenter
from classes.papila_builders import build_papila_clinical
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Evaluate classifier on Papila with UNet crops")
parser.add_argument("--filtered-metrics", type=Path, required=True, help="CSV of Papila samples with acceptable Dice")
parser.add_argument("--segmenter-manifest", type=Path, required=True, help="Manifest used to train the UNet segmenter")
parser.add_argument("--segmenter-weights", type=Path, required=True, help="Path to trained UNet weights (best.pt)")
parser.add_argument("--classifier-weights", type=Path, required=False, help="Path to classifier checkpoint (refuge_classifier_best.pt)")
parser.add_argument("--refuge-root", type=Path, default=Path("REFUGE"))
parser.add_argument("--image-dir", type=Path, default=Path("Papila/FundusImages"))
parser.add_argument("--clinical-dir", type=Path, default=Path("Papila/ClinicalData"))
parser.add_argument("--label-col", type=str, default="Diagnosis", help="Column name holding Papila labels")
parser.add_argument(
"--positive-labels",
nargs="*",
default=["glaucoma", "glaucoma suspect", "suspect"],
help="Values treated as glaucoma-positive when labels are non-numeric",
)
parser.add_argument("--dice-threshold", type=float, default=0.01, help="Minimum Dice (disc or cup) to keep a sample")
parser.add_argument("--segmenter-threshold", type=float, default=0.5, help="Probability threshold for UNet geometry")
parser.add_argument("--segmenter-normalize", choices=["none", "imagenet", "per_image"], default="per_image")
parser.add_argument("--segmenter-tta", action="store_true", help="Enable TTA (H/V flips) when deriving geometry")
parser.add_argument("--crop-scale", type=float, default=2.5)
parser.add_argument("--crop-size", type=int, default=224)
parser.add_argument("--batch-size", type=int, default=32)
parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
parser.add_argument("--output", type=Path, default=None, help="Optional CSV to store per-sample probabilities")
parser.add_argument(
"--cache-dir",
type=Path,
default=Path("analysis_data/classifier_cache"),
help="Directory to reuse classifier preprocessing cache",
)
parser.add_argument(
"--use-gt-masks",
action="store_true",
help="Use ground truth Papila contours instead of UNet predictions",
)
parser.add_argument(
"--gt-contours-dir",
type=Path,
default=Path("Papila/ExpertsSegmentations/Contours"),
help="Directory containing Papila contour text files",
)
parser.add_argument(
"--backbone",
type=str,
default=None,
help="Optional backbone name (e.g. inception_v3, densenet121). Requires matching classifier weights.",
)
return parser.parse_args()
def load_allowed_ids(path: Path, dice_threshold: float) -> Set[str]:
allowed: Set[str] = set()
with path.open(newline="") as fp:
reader = csv.DictReader(fp)
for row in reader:
sample_id = row.get("sample_id")
if not sample_id or sample_id == "__mean__":
continue
try:
disc = float(row.get("dice_disc", "nan"))
cup = float(row.get("dice_cup", "nan"))
except ValueError:
continue
if disc < dice_threshold and cup < dice_threshold:
continue
allowed.add(sample_id)
return allowed
def build_papila_samples(
image_dir: Path,
clinical_dir: Path,
label_col: str,
positive_labels: Sequence[str],
allowed_ids: Set[str],
) -> List[RefugeSample]:
clinical = build_papila_clinical(
image_dir=str(image_dir),
clinical_dir=str(clinical_dir),
label_col=label_col,
cat_cols=[],
)
positives = {lbl.lower() for lbl in positive_labels}
samples: Dict[str, RefugeSample] = {}
for _, row in clinical.df.iterrows():
image_path = clinical.get_image_path(row)
sample_id = f"papila_{image_path.stem}"
if sample_id not in allowed_ids or sample_id in samples:
continue
value = row.get(label_col)
if value is None or (isinstance(value, float) and np.isnan(value)):
continue
label: Optional[int]
try:
label_int = int(value)
if label_int == 2:
continue
label = 1 if label_int > 0 else 0
except (TypeError, ValueError):
label = 1 if str(value).strip().lower() in positives else 0
samples[sample_id] = RefugeSample(
sample_id=sample_id,
dataset="papila",
split="eval",
image_path=Path(image_path),
label=label,
device=None,
mask_path=None,
fovea_coord=None,
)
return list(samples.values())
def load_contour(path: Path) -> np.ndarray:
coords = np.loadtxt(path)
if coords.ndim == 1:
coords = coords.reshape(-1, 2)
return coords
def contour_to_mask(coords: np.ndarray, size: Sequence[int]) -> np.ndarray:
if coords is None or coords.size == 0:
return np.zeros((size[1], size[0]), dtype=np.uint8)
img = Image.new("L", size, 0)
draw = ImageDraw.Draw(img)
points = [tuple(map(float, pt)) for pt in coords]
draw.polygon(points, outline=1, fill=1)
return np.array(img, dtype=np.uint8)
class PapilaGTGeometryProvider:
def __init__(self, contours_dir: Path) -> None:
self.contours_dir = contours_dir
def _pick(self, base: str, kind: str) -> Optional[Path]:
for exp in ("exp2", "exp1"):
cand = self.contours_dir / f"{base}_{kind}_{exp}.txt"
if cand.exists():
return cand
return None
def __call__(self, sample: RefugeSample, scale: float):
base = Path(sample.image_path).stem
disc_path = self._pick(base, "disc")
cup_path = self._pick(base, "cup")
if disc_path is None or cup_path is None:
raise RuntimeError(f"Missing ground-truth contours for {sample.sample_id}")
image = Image.open(sample.image_path).convert("RGB")
disc_coords = load_contour(disc_path)
cup_coords = load_contour(cup_path)
disc_mask = contour_to_mask(disc_coords, image.size)
cup_mask = contour_to_mask(cup_coords, image.size)
cup_mask = ((cup_mask > 0) & (disc_mask > 0)).astype(np.uint8)
geom = _geometry_from_mask(disc_mask, scale)
return geom, disc_mask.astype(np.uint8), cup_mask.astype(np.uint8)
def build_backbone(name: Optional[str]) -> Optional[torch.nn.Module]:
if not name:
return None
key = name.lower()
if key not in BACKBONES:
raise ValueError(f"Unknown backbone '{name}'. Available: {', '.join(sorted(BACKBONES.keys()))}")
spec = BACKBONES[key]
model = spec.ctor(weights=spec.weights_default)
out_dim, model = spec.strip(model)
setattr(model, "_feature_dim", out_dim)
if key == "refugelike":
load_backbone_weights(key, model)
return model
def evaluate_records(
clf: RefugeClassification,
records: Sequence[RefugeClassificationRecord],
device: str,
batch_size: int,
) -> Dict[str, float]:
dataset = RefugeClassificationDataset(
records,
transform=clf.eval_transform,
polar_transform=clf.polar_transform,
size=clf.crop_size,
)
loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=0)
clf.backbone.to(device).eval()
clf.classifier_head.to(device).eval()
preds: List[float] = []
targets: List[int] = []
with torch.no_grad():
for batch in tqdm(loader, desc="Papila Eval", leave=False, unit="batch"):
images = batch["image"].to(device)
polars = batch["polar"].to(device)
extra_feats = batch["features"].to(device)
labels = batch["label"].cpu().numpy().tolist()
feats_img = clf.backbone(images)
feats = feats_img
if clf.use_polar:
feats_polar = clf.backbone(polars)
feats = torch.cat([feats, feats_polar], dim=1)
if clf.extra_feature_dim > 0:
feats = torch.cat([feats, extra_feats], dim=1)
logits = clf.classifier_head(feats)
probs = torch.softmax(logits, dim=1)[:, 1].cpu().numpy().tolist()
preds.extend(probs)
targets.extend(labels)
metrics: Dict[str, float] = {"count": float(len(targets))}
unique_labels = set(targets)
if len(unique_labels) >= 2:
metrics["auc"] = float(torchmetrics_auc(targets, preds))
else:
metrics["auc"] = float("nan")
preds_bin = [1 if p >= 0.5 else 0 for p in preds]
accuracy = sum(int(p == t) for p, t in zip(preds_bin, targets)) / max(1, len(targets))
metrics["accuracy"] = float(accuracy)
metrics["mean_prob"] = float(np.mean(preds)) if preds else float("nan")
metrics["labels_pos"] = float(sum(targets))
if preds:
metrics["probs_std"] = float(np.std(preds))
return metrics
def torchmetrics_auc(targets: Sequence[int], preds: Sequence[float]) -> float:
try:
from sklearn.metrics import roc_auc_score
except ImportError as exc:
raise RuntimeError("scikit-learn is required to compute AUC") from exc
return float(roc_auc_score(targets, preds))
def main() -> None:
args = parse_args()
device = args.device
allowed_ids = load_allowed_ids(args.filtered_metrics, args.dice_threshold)
if not allowed_ids:
raise SystemExit("No Papila samples passed the Dice threshold.")
papila_samples = build_papila_samples(
args.image_dir,
args.clinical_dir,
args.label_col,
args.positive_labels,
allowed_ids,
)
if not papila_samples:
raise SystemExit("No Papila samples with labels matched the filtered metrics.")
cache_dir = args.cache_dir
if args.use_gt_masks and cache_dir is not None:
cache_dir = cache_dir / "gt"
if args.use_gt_masks:
geometry_provider = PapilaGTGeometryProvider(args.gt_contours_dir)
segmenter = None
else:
segmenter = UNetSegmenter(
manifest_path=args.segmenter_manifest,
device=device,
normalize=args.segmenter_normalize,
)
seg_state = torch.load(args.segmenter_weights, map_location=device)
seg_state_dict = seg_state.get("model", seg_state)
segmenter.model.load_state_dict(seg_state_dict)
segmenter.model.to(device)
geometry_provider = UNetGeometryProvider(
segmenter=segmenter,
threshold=args.segmenter_threshold,
tta=args.segmenter_tta,
)
pre = RefugePreprocessing(args.refuge_root)
dummy_seg = RefugeSegmentation(pre)
backbone = build_backbone(args.backbone)
clf = RefugeClassification(
pre,
dummy_seg,
geometry_fn=geometry_provider,
cache_dir=cache_dir,
backbone=backbone,
)
clf.crop_scale = args.crop_scale
clf.crop_size = args.crop_size
clf.eval_transform = _default_image_transform(args.crop_size)
clf.ttt_transform = clf.eval_transform
if args.classifier_weights is not None:
clf_state = torch.load(args.classifier_weights, map_location=device)
clf.backbone.load_state_dict(clf_state["backbone"])
clf.classifier_head.load_state_dict(clf_state["classifier"])
clf.rotation_head.load_state_dict(clf_state["rotation"])
if "feature_reg" in clf_state and getattr(clf, "feature_reg_head", None) is not None:
clf.feature_reg_head.load_state_dict(clf_state["feature_reg"])
records = clf.build_records_for_samples(
papila_samples,
crop_scale=args.crop_scale,
progress_prefix="papila_eval",
)
if not records:
raise SystemExit("Unable to build any records; check geometry predictions or labels.")
metrics = evaluate_records(clf, records, device=device, batch_size=args.batch_size)
print(f"Samples evaluated: {int(metrics['count'])}")
print(f"AUC: {metrics['auc']:.4f}" if not np.isnan(metrics['auc']) else "AUC: NaN")
print(f"Accuracy @0.5: {metrics['accuracy']:.4f}")
print(f"Mean glaucoma prob: {metrics['mean_prob']:.4f}")
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
with args.output.open("w", newline="") as fp:
writer = csv.writer(fp)
writer.writerow(["sample_id", "prob_glaucoma", "label"])
clf.backbone.eval()
clf.classifier_head.eval()
dataset = RefugeClassificationDataset(
records,
transform=clf.eval_transform,
polar_transform=clf.polar_transform,
size=clf.crop_size,
)
loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=False, num_workers=0)
with torch.no_grad():
for batch in tqdm(loader, desc="Papila Output", leave=False, unit="batch"):
images = batch["image"].to(device)
polars = batch["polar"].to(device)
extra_feats = batch["features"].to(device)
ids = batch["sample_id"]
labels = batch["label"].tolist()
feats_img = clf.backbone(images)
feats = feats_img
if clf.use_polar:
feats_polar = clf.backbone(polars)
feats = torch.cat([feats, feats_polar], dim=1)
if clf.extra_feature_dim > 0:
feats = torch.cat([feats, extra_feats], dim=1)
logits = clf.classifier_head(feats)
probs = torch.softmax(logits, dim=1)[:, 1].cpu().numpy().tolist()
for sid, prob, label in zip(ids, probs, labels):
writer.writerow([sid, prob, label])
print(f"Per-sample probabilities written to {args.output}")
if __name__ == "__main__":
main()
scripts/legacy/extract_best_auc.py
-115
View File
@@ -1,115 +0,0 @@
#!/usr/bin/env python3
"""
Quick utility to recover the best epoch metrics from HyperTower run folders.
Example:
python scripts/extract_best_auc.py analysis_data/img_only_densenet_gt_bin/img_only_densenet_gt_bin_20251028_112733
By default it looks for columns named like `auc_fused` (set via --metric) inside each
`fold{n}_epoch_log.csv`, returning the epoch with the highest value plus the holdout
metrics, if present.
"""
from __future__ import annotations
import argparse
import csv
import json
import math
from pathlib import Path
from typing import Dict, Optional, Tuple
def to_float(value: Optional[str]) -> Optional[float]:
if value is None:
return None
value = value.strip()
if not value:
return None
try:
out = float(value)
except ValueError:
return None
if math.isnan(out):
return None
return out
def best_row(path: Path, metric: str) -> Optional[Dict[str, str]]:
if not path.exists():
return None
best: Optional[Tuple[float, int, Dict[str, str]]] = None
with path.open("r", newline="") as fp:
reader = csv.DictReader(fp)
for row in reader:
val = to_float(row.get(metric))
if val is None:
continue
epoch = int(row.get("epoch", reader.line_num))
if best is None or val > best[0]:
best = (val, epoch, row)
return best[2] if best else None
def summarize_fold(row: Dict[str, str], metric: str) -> Dict[str, float]:
data: Dict[str, float] = {}
for key in (metric, f"holdout_{metric.split('_', 1)[-1]}", "holdout_auc_img", "holdout_auc_fused"):
val = to_float(row.get(key))
if val is not None:
data[key] = val
epoch_val = to_float(row.get("epoch"))
if epoch_val is not None:
data["epoch"] = int(epoch_val)
return data
def main() -> None:
ap = argparse.ArgumentParser(description="Extract best-per-fold metric from HyperTower runs.")
ap.add_argument("run_dir", type=Path, help="Run directory (contains fold*_epoch_log.csv)")
ap.add_argument("--metric", default="auc_fused", help="Metric column to maximise (default: auc_fused)")
ap.add_argument("--json", type=Path, default=None, help="Optional path to dump JSON summary")
args = ap.parse_args()
run_dir: Path = args.run_dir
metric: str = args.metric
if not run_dir.exists():
raise SystemExit(f"Run directory not found: {run_dir}")
fold_summaries: Dict[str, Dict[str, float]] = {}
metric_values = []
for csv_path in sorted(run_dir.glob("fold*_epoch_log.csv")):
best = best_row(csv_path, metric)
fold_name = csv_path.stem.replace("_epoch_log", "")
if best is None:
print(f"{fold_name}: no valid '{metric}' values found")
continue
summary = summarize_fold(best, metric)
fold_summaries[fold_name] = summary
val = summary.get(metric)
if val is not None:
metric_values.append(val)
holdout_val = summary.get(f"holdout_{metric.split('_', 1)[-1]}")
print(f"{fold_name}: epoch={summary.get('epoch')} {metric}={val:.4f}" if val is not None else f"{fold_name}: epoch={summary.get('epoch')}")
if holdout_val is not None:
print(f" holdout_{metric.split('_', 1)[-1]}={holdout_val:.4f}")
if metric_values:
mean_val = sum(metric_values) / len(metric_values)
print(f"\nMean best {metric}: {mean_val:.4f}")
if args.json:
payload = {
"run_dir": str(run_dir),
"metric": metric,
"folds": fold_summaries,
"mean_metric": (sum(metric_values) / len(metric_values)) if metric_values else None,
}
args.json.parent.mkdir(parents=True, exist_ok=True)
args.json.write_text(json.dumps(payload, indent=2))
print(f"Summary written to {args.json}")
if __name__ == "__main__":
main()
scripts/legacy/paper_metadata.py
-493
View File
@@ -1,493 +0,0 @@
import pandas as pd
from classes import HyperTower, ClinicalData, list_names, build_papila_clinical
from pathlib import Path
import shutil, json, textwrap
from datetime import datetime
import numpy as np
from typing import Dict, List, Tuple
from sklearn.model_selection import GroupKFold
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.metrics import roc_auc_score
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.preprocessing import label_binarize
def _proba_from_model(model, X):
if hasattr(model[-1], "predict_proba"):
return model.predict_proba(X)
dec = model.decision_function(X)
if dec.ndim == 1: # binary margins -> make 2-col
dec = np.stack([-dec, dec], axis=1)
e = np.exp(dec - dec.max(axis=1, keepdims=True))
return e / e.sum(axis=1, keepdims=True)
def _cv_auc_multiclass_per_class(X, y, groups, model, n_splits=5) -> np.ndarray:
"""
Returns a length-3 array of mean OvR AUCs for Class0/1/2 across GroupKFold.
Uses nan-safe means if a class is absent in a fold's test split.
"""
gkf = GroupKFold(n_splits=n_splits)
per_class_lists = [[], [], []]
for tr, te in gkf.split(X, y, groups):
model.fit(X[tr], y[tr])
proba = _proba_from_model(model, X[te])
y_te = y[te]
y_bin = label_binarize(y_te, classes=[0, 1, 2]) # (n,3)
for k in range(3):
yk = y_bin[:, k]
if yk.min() != yk.max(): # both classes present
per_class_lists[k].append(roc_auc_score(yk, proba[:, k]))
else:
per_class_lists[k].append(np.nan)
return np.array([np.nanmean(per_class_lists[k]) for k in range(3)], dtype=float)
def _cv_auc_binary(X, y, groups, model, n_splits=5) -> float:
mask = np.isin(y, [0, 1])
Xb, yb, gb = X[mask], y[mask], groups[mask]
gkf = GroupKFold(n_splits=n_splits)
aucs = []
for tr, te in gkf.split(Xb, yb, gb):
model.fit(Xb[tr], yb[tr])
if hasattr(model[-1], "predict_proba"):
p = model.predict_proba(Xb[te])[:, 1]
else:
p = model.decision_function(Xb[te])
# logistic squash for safety
if np.ptp(p) > 0:
p = 1.0 / (1.0 + np.exp(-p))
else:
p = np.full_like(p, 0.5, dtype=float)
# only compute if both classes present
if len(np.unique(yb[te])) == 2:
aucs.append(roc_auc_score(yb[te], p))
else:
aucs.append(np.nan)
return float(np.nanmean(aucs))
# -----------------------------------
# 1) Build Clinical Data (paper-faithful)
# -----------------------------------
IMAGE_DIR = "Papila/FundusImages"
CLINICAL_DIR = "Papila/ClinicalData"
LABEL_COL = "Diagnosis"
CAT_COLS = ["Gender", "Phakic/Pseudophakic"]
paper_auc = {
"TEST3_multiclass": { # Class0=Healthy, Class1=Glaucoma, Class2=Suspect
"LogReg": {"Class0": 0.67, "Class1": 0.66, "Class2": 0.67}, # from Fig. 7 (rounded)
"kNN": {"Class0": 0.72, "Class1": 0.70, "Class2": 0.76}, # your read of Fig. 7
"RF": {"Class0": 0.66, "Class1": 0.66, "Class2": 0.67}, # from Fig. 7 (rounded)
"SVM": {"Class0": 0.66, "Class1": 0.65, "Class2": 0.66}, # from Fig. 7 (rounded)
},
"TEST4_binary": { # Healthy vs Glaucoma (Suspects removed)
"LogReg": 0.71, # from text/Fig. 7 range midpoint
"kNN": 0.75, # your read of Fig. 7
"RF": 0.70, # from Fig. 7 (rounded)
"SVM": 0.69, # from Fig. 7 (rounded)
}
}
clinical = build_papila_clinical(
image_dir=IMAGE_DIR,
clinical_dir=CLINICAL_DIR,
label_col=LABEL_COL,
cat_cols=CAT_COLS,
)
# -----------------------------------
# 2) Feature matrix (no MD; IOP_corr already present)
# -----------------------------------
def build_feature_matrix(clinical) -> Tuple[np.ndarray, np.ndarray, np.ndarray, List[str]]:
"""
Returns:
X: features (N x D)
y: labels (Diagnosis: 0 healthy, 1 glaucoma, 2 suspect)
groups: patient IDs for GroupKFold
feat_names: list of feature names in X order
"""
df = clinical.df.copy()
# Scalars used in paper-style baselines (no VF_MD)
scalars = ["Age", "dioptre_1", "dioptre_2", "astigmatism",
"Pachymetry", "Axial_Length", "IOP_corr"]
# Categorical one-hot
cats = ["Gender", "Phakic/Pseudophakic"]
df_cats = pd.get_dummies(df[cats].astype("category"), drop_first=False, prefix=cats)
# Combine
X = pd.concat([df[scalars], df_cats], axis=1)
# Median impute numerics (simple, consistent)
for c in scalars:
med = pd.to_numeric(X[c], errors="coerce").median()
X[c] = pd.to_numeric(X[c], errors="coerce").fillna(med)
y = df[LABEL_COL].astype(int).values
groups = df["Patient ID"].astype(int).values
feat_names = list(X.columns)
return X.values.astype(np.float32), y, groups, feat_names
# ----------------------------
# 3) Model zoo (the four methods used in the paper)
# ----------------------------
def make_models(best_params: dict | None = None, random_state: int = 42) -> dict:
"""
Build paper-like baseline models. If best_params is provided (a dict mapping
model-name -> param dict with pipeline-style keys like 'clf__C'), those
params are applied to the corresponding pipelines.
"""
models = {
"LogReg": Pipeline([
("scaler", StandardScaler()),
("clf", LogisticRegression(
max_iter=100,
solver="lbfgs",
multi_class="auto"
))
]),
"kNN": Pipeline([
("scaler", StandardScaler()),
("clf", KNeighborsClassifier(
n_neighbors=5,
weights="uniform",
metric="minkowski",
p=2
))
]),
"RF": Pipeline([
("clf", RandomForestClassifier(
n_estimators=100,
criterion="gini",
max_depth=None,
min_samples_split=2,
min_samples_leaf=1,
max_features="sqrt",
bootstrap=True,
# random_state left as default; set via best_params if desired
))
]),
"SVM": Pipeline([
("scaler", StandardScaler()),
("clf", SVC(
C=1.0,
kernel="rbf",
gamma="scale",
probability=False
))
]),
}
# Apply overrides if provided
if best_params:
for name, params in best_params.items():
if name in models and params:
models[name].set_params(**params)
return models
# -----------------------------------
# 4) CV AUCs (mean over 5 folds; GroupKFold by patient)
# -----------------------------------
def _cv_auc_multiclass(X, y, groups, model, n_splits=5) -> float:
gkf = GroupKFold(n_splits=n_splits)
aucs = []
for tr, te in gkf.split(X, y, groups):
model.fit(X[tr], y[tr])
if hasattr(model[-1], "predict_proba"):
proba = model.predict_proba(X[te])
else:
dec = model.decision_function(X[te])
if dec.ndim == 1:
dec = np.stack([-dec, dec], axis=1)
e = np.exp(dec - dec.max(axis=1, keepdims=True))
proba = e / e.sum(axis=1, keepdims=True)
aucs.append(roc_auc_score(y[te], proba, multi_class="ovr", average="macro"))
return float(np.mean(aucs))
def _cv_auc_binary(X, y, groups, model, n_splits=5) -> float:
# Keep classes 0 (healthy) and 1 (glaucoma); drop suspects (2)
mask = np.isin(y, [0, 1])
Xb, yb, gb = X[mask], y[mask], groups[mask]
gkf = GroupKFold(n_splits=n_splits)
aucs = []
for tr, te in gkf.split(Xb, yb, gb):
model.fit(Xb[tr], yb[tr])
if hasattr(model[-1], "predict_proba"):
p = model.predict_proba(Xb[te])[:, 1]
else:
p = model.decision_function(Xb[te])
# simple logistic squashing if needed
if np.ptp(p) > 0:
p = 1.0 / (1.0 + np.exp(-p))
else:
p = np.full_like(p, 0.5, dtype=float)
aucs.append(roc_auc_score(yb[te], p))
return float(np.mean(aucs))
# -----------------------------------
# 5) Run both tests (multiclass + binary) and print table
# -----------------------------------
def run_papila_clinical_baselines(clinical, n_splits: int = 5,
random_state: int = 42,
best_params: dict | None = None) -> pd.DataFrame:
X, y, groups, feat_names = build_feature_matrix(clinical)
models = make_models(best_params=best_params, random_state=random_state)
rows = []
for name, model in models.items():
c0, c1, c2 = _cv_auc_multiclass_per_class(X, y, groups, model, n_splits=n_splits)
auc_bin = _cv_auc_binary(X, y, groups, model, n_splits=n_splits)
rows.append({"model": name, "Class0": c0, "Class1": c1, "Class2": c2, "Binary": auc_bin})
df = pd.DataFrame(rows).set_index("model").sort_index()
return df
results = run_papila_clinical_baselines(clinical, n_splits=5)
# print(results.to_string(float_format=lambda x: f"{x:.3f}"))
##############################
from sklearn.model_selection import ParameterGrid
from sklearn.base import clone
from sklearn.preprocessing import label_binarize
from sklearn.utils import check_random_state
# ==============================
# Helper: per-class & binary AUC with GroupKFold
# ==============================
def _proba_from_model(model, X):
if hasattr(model[-1], "predict_proba"):
return model.predict_proba(X)
# decision_function fallback
dec = model.decision_function(X)
if dec.ndim == 1: # binary margin -> 2-col probs
dec = np.stack([-dec, dec], axis=1)
e = np.exp(dec - dec.max(axis=1, keepdims=True))
return e / e.sum(axis=1, keepdims=True)
def _cv_auc_perclass_and_binary(X, y, groups, model, n_splits=5):
"""
Returns:
per_class_auc: length-3 array (Class0, Class1, Class2) averaged over folds
binary_auc: scalar (0 vs 1) averaged over folds
"""
gkf = GroupKFold(n_splits=n_splits)
# Hold fold-wise per-class AUCs (list of arrays of length 3)
perclass_fold_scores = []
binary_fold_scores = []
for tr, te in gkf.split(X, y, groups):
y_te = y[te]
# Multiclass per-class (OvR)
model.fit(X[tr], y[tr])
proba = _proba_from_model(model, X[te])
# One-vs-rest per-class AUCs (skip a class if absent in test fold)
y_bin = label_binarize(y_te, classes=[0, 1, 2]) # shape (n, 3)
perclass_scores = []
for k in range(3):
yk = y_bin[:, k]
# Only compute if both 0 and 1 are present
if yk.min() != yk.max():
perclass_scores.append(roc_auc_score(yk, proba[:, k]))
else:
perclass_scores.append(np.nan)
perclass_fold_scores.append(perclass_scores)
# Binary AUC (0 vs 1; drop class 2)
mask = np.isin(y_te, [0, 1])
if mask.sum() > 0 and len(np.unique(y_te[mask])) == 2:
# we need probabilities/margins for class 1 among (0,1)
# Map proba[:, 1] if the model was trained 3-way; we restrict te samples to 0/1
binary_p = proba[mask, 1]
binary_y = y_te[mask]
binary_fold_scores.append(roc_auc_score(binary_y, binary_p))
else:
binary_fold_scores.append(np.nan)
# Average over folds (ignore NaNs if a class was missing in a fold)
perclass_arr = np.array(perclass_fold_scores, dtype=float) # (n_folds, 3)
per_class_auc = np.nanmean(perclass_arr, axis=0)
binary_auc = float(np.nanmean(np.array(binary_fold_scores, dtype=float)))
return per_class_auc, binary_auc
# ==============================
# Distance-to-paper objective
# ==============================
def _distance_to_paper(model_name: str,
per_class_auc: np.ndarray,
binary_auc: float,
paper_auc: Dict,
w_mc: float = 1.0,
w_bin: float = 1.0) -> float:
mc_targets = paper_auc["TEST3_multiclass"][model_name]
tvec = np.array([mc_targets["Class0"], mc_targets["Class1"], mc_targets["Class2"]], dtype=float)
mc_diff = np.nanmean(np.abs(per_class_auc - tvec)) # mean absolute difference over 3 classes
bin_target = paper_auc["TEST4_binary"][model_name]
bin_diff = abs(binary_auc - bin_target)
return float(w_mc * mc_diff + w_bin * bin_diff)
# ==============================
# Parameter grids (paper-ish, not crazy-large)
# ==============================
def get_param_grids() -> Dict[str, List[dict]]:
return {
"LogReg": [
{
"clf__C": [0.01, 0.1, 1.0, 3.0, 10.0],
"clf__class_weight": [None, "balanced"],
"clf__max_iter": [200, 500],
# lbfgs + l2 is implied
}
],
"kNN": [
{
"clf__n_neighbors": [3, 5, 7, 9, 11],
"clf__weights": ["uniform", "distance"],
"clf__p": [1, 2], # Manhattan vs Euclidean
}
],
"RF": [
{
"clf__n_estimators": [200, 500, 1000],
"clf__max_depth": [None, 5, 10, 20],
"clf__max_features": ["sqrt", "log2", 0.5],
"clf__min_samples_leaf": [1, 2, 5],
"clf__class_weight": [None, "balanced"],
# If you want determinism add: "clf__random_state": [42]
}
],
"SVM": [
{
"clf__C": [0.1, 1.0, 3.0, 10.0],
"clf__gamma": ["scale", "auto", 0.1, 0.01, 0.001],
"clf__kernel": ["rbf"], # fixed to rbf as in paper-like default
}
],
}
# ==============================
# Grid search loop minimizing distance-to-paper
# ==============================
def search_params_to_match_paper(
clinical,
models: Dict[str, Pipeline],
paper_auc: Dict,
n_splits: int = 5,
w_mc: float = 1.0,
w_bin: float = 1.0,
verbose: bool = True,
) -> Tuple[pd.DataFrame, Dict[str, dict]]:
X, y, groups, feat_names = build_feature_matrix(clinical)
grids = get_param_grids()
summary_rows = []
best_params_by_model = {}
for name, base_model in models.items():
if name not in grids:
if verbose:
print(f"[warn] No grid for {name}, skipping.")
continue
best_loss = np.inf
best_params = None
best_mc = None
best_bin = None
for param_set in ParameterGrid(grids[name]):
model = clone(base_model).set_params(**param_set)
per_class_auc, binary_auc = _cv_auc_perclass_and_binary(
X, y, groups, model, n_splits=n_splits
)
loss = _distance_to_paper(
name, per_class_auc, binary_auc, paper_auc, w_mc=w_mc, w_bin=w_bin
)
if verbose:
mc_str = " / ".join(f"{a:.3f}" if np.isfinite(a) else "nan" for a in per_class_auc)
print(f"[{name}] params={param_set} | mc per-class={mc_str} | bin={binary_auc:.3f} | loss={loss:.4f}")
if loss < best_loss:
best_loss = loss
best_params = param_set
best_mc = per_class_auc
best_bin = binary_auc
# store
best_params_by_model[name] = best_params
summary_rows.append({
"model": name,
"best_loss": best_loss,
"best_params": json.dumps(best_params),
"mc_Class0": float(best_mc[0]),
"mc_Class1": float(best_mc[1]),
"mc_Class2": float(best_mc[2]),
"binary_auc": float(best_bin),
"paper_mc_Class0": paper_auc["TEST3_multiclass"][name]["Class0"],
"paper_mc_Class1": paper_auc["TEST3_multiclass"][name]["Class1"],
"paper_mc_Class2": paper_auc["TEST3_multiclass"][name]["Class2"],
"paper_binary": paper_auc["TEST4_binary"][name],
})
df = pd.DataFrame(summary_rows).set_index("model").sort_values("best_loss")
return df, best_params_by_model
# ==============================
# Run the search
# ==============================
models = make_models(random_state=42)
df_match, best_params = search_params_to_match_paper(
clinical=clinical,
models=models,
paper_auc=paper_auc,
n_splits=5,
w_mc=1.0, # weight multiclass distance
w_bin=1.0, # weight binary distance
verbose=True
)
# print("\n=== Best params found (by minimal distance-to-paper) ===")
# print(df_match[["best_loss","best_params","mc_Class0","mc_Class1","mc_Class2","binary_auc",
# "paper_mc_Class0","paper_mc_Class1","paper_mc_Class2","paper_binary"]])
# print("\nBest param dicts:")
for k, v in best_params.items():
print(k, "->", v)
results2 = run_papila_clinical_baselines(clinical, n_splits=5, random_state=42, best_params=best_params)
print(f"Default Settings: {results.round(2)}")
print(f"Best Params Settings: {results2.round(2)}")
print(f" Paper Results: {pd.DataFrame({
model: {**vals, "Binary": paper_auc["TEST4_binary"][model]}
for model, vals in paper_auc["TEST3_multiclass"].items()
}).T[["Class0","Class1","Class2","Binary"]]}")
scripts/legacy/run_all_sweep.sh
-119
View File
@@ -1,119 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Usage:
# bash scripts/run_all_sweep.sh --epochs 25 --n-splits 5 --batch-size 8 [extra args]
#
# Merged sweep: runs the SE attention grid (bridge/tower/both × R=8/16/32),
# skipping tower-only non-normalized variants (tower normalization is a no-op),
# and includes binary eval counterparts for each baseline run. It also submits
# the full gradual-thaw grid (multiclass + binary variants).
ARGS=("$@")
run() {
local SHORT="$1"; shift
echo "=== Running: $SHORT ==="
# Skip if a summary for this shortname already exists
if ls "analysis_data/${SHORT}_"*.md >/dev/null 2>&1; then
echo "… skipping ${SHORT} (summary already present)"
return 0
fi
python3 scripts/run_multifold.py \
--shortname "$SHORT" \
"$@" \
"${ARGS[@]}" || true
}
echo "--- SE Grid (bridge/tower/both × R=8/16/32; tower nonorm skipped) ---"
for R in 8 16 32; do
# Bridge-only
run "se_bridge_R${R}_norm" --se-where bridge --se-reduction ${R} --se-pre-norm --checkpoint-best
run "se_bridge_R${R}_norm_bin" --se-where bridge --se-reduction ${R} --se-pre-norm --checkpoint-best --eval_mode binary
run "se_bridge_R${R}_nonorm" --se-where bridge --se-reduction ${R} --no-se-pre-norm --checkpoint-best
run "se_bridge_R${R}_nonorm_bin" --se-where bridge --se-reduction ${R} --no-se-pre-norm --checkpoint-best --eval_mode binary
# Tower-only
run "se_tower_R${R}_norm" --se-where tower --se-reduction-tower ${R} --se-pre-norm-tower --checkpoint-best
run "se_tower_R${R}_norm_bin" --se-where tower --se-reduction-tower ${R} --se-pre-norm-tower --checkpoint-best --eval_mode binary
# Tower+Bridge
run "se_tower_bridge_R${R}_norm" \
--se-where both --se-reduction ${R} --se-reduction-tower ${R} \
--se-pre-norm --se-pre-norm-tower --checkpoint-best
run "se_tower_bridge_R${R}_norm_bin" \
--se-where both --se-reduction ${R} --se-reduction-tower ${R} \
--se-pre-norm --se-pre-norm-tower --checkpoint-best --eval_mode binary
run "se_tower_bridge_R${R}_nonorm" \
--se-where both --se-reduction ${R} --se-reduction-tower ${R} \
--no-se-pre-norm --no-se-pre-norm-tower --checkpoint-best
run "se_tower_bridge_R${R}_nonorm_bin" \
--se-where both --se-reduction ${R} --se-reduction-tower ${R} \
--no-se-pre-norm --no-se-pre-norm-tower --checkpoint-best --eval_mode binary
done
THAW_COMMON_ARGS=(
--gradual-thaw
--thaw-phase-duration 5
--thaw-ratio 0.33
--thaw-start-epoch 5
--early-stop
--early-patience 5
)
echo "--- Gradual Thaw Grid (multiclass + binary) ---"
# Bridge-only thaw runs (norm and nonorm)
for R in 8 16 32; do
for MODE in norm nonorm; do
if [[ "$MODE" == "norm" ]]; then
FLAGS=(--se-where bridge --se-reduction "$R" --se-pre-norm --checkpoint-best)
else
FLAGS=(--se-where bridge --se-reduction "$R" --no-se-pre-norm --checkpoint-best)
fi
run "thaw_se_bridge_R${R}_${MODE}" "${FLAGS[@]}" "${THAW_COMMON_ARGS[@]}"
run "thawbin_se_bridge_R${R}_${MODE}" "${FLAGS[@]}" "${THAW_COMMON_ARGS[@]}" --eval_mode binary
done
done
# Tower-only thaw runs (norm and nonorm)
for R in 8 16 32; do
for MODE in norm nonorm; do
if [[ "$MODE" == "norm" ]]; then
FLAGS=(--se-where tower --se-reduction-tower "$R" --se-pre-norm-tower --checkpoint-best)
else
FLAGS=(--se-where tower --se-reduction-tower "$R" --no-se-pre-norm-tower --checkpoint-best)
fi
run "thaw_se_tower_R${R}_${MODE}" "${FLAGS[@]}" "${THAW_COMMON_ARGS[@]}"
run "thawbin_se_tower_R${R}_${MODE}" "${FLAGS[@]}" "${THAW_COMMON_ARGS[@]}" --eval_mode binary
done
done
# Tower+bridge thaw runs (norm and nonorm)
for R in 8 16 32; do
for MODE in norm nonorm; do
if [[ "$MODE" == "norm" ]]; then
FLAGS=(
--se-where both
--se-reduction "$R"
--se-reduction-tower "$R"
--se-pre-norm
--se-pre-norm-tower
--checkpoint-best
)
else
FLAGS=(
--se-where both
--se-reduction "$R"
--se-reduction-tower "$R"
--no-se-pre-norm
--no-se-pre-norm-tower
--checkpoint-best
)
fi
run "thaw_se_tower_bridge_R${R}_${MODE}" "${FLAGS[@]}" "${THAW_COMMON_ARGS[@]}"
run "thawbin_se_tower_bridge_R${R}_${MODE}" "${FLAGS[@]}" "${THAW_COMMON_ARGS[@]}" --eval_mode binary
done
done
echo "Merged sweep submitted. Check analysis_data/* and models/* for outputs."
scripts/legacy/run_gradual_thaw_top5.sh
-79
View File
@@ -1,79 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Usage:
# bash scripts/run_gradual_thaw_top5.sh --epochs 20 --n-splits 5 --batch-size 8 [extra args]
#
# Runs the full gradual-thaw grid aligned with the SE sweep (bridge/tower/both × R=8/16/32 × norm vs nonorm).
ARGS=("$@")
run() {
local SHORT="$1"; shift
echo "=== Running: $SHORT ==="
if ls "analysis_data/${SHORT}_"*.md >/dev/null 2>&1; then
echo "… skipping ${SHORT} (summary already present)"
return 0
fi
python3 scripts/run_multifold.py \
--shortname "$SHORT" \
--gradual-thaw --thaw-phase-duration 5 --thaw-ratio 0.33 --thaw-start-epoch 5 \
--early-stop --early-patience 5 \
"$@" \
"${ARGS[@]}" || true
}
# Bridge-only thaw runs
for R in 8 16 32; do
for MODE in norm nonorm; do
if [[ "$MODE" == "norm" ]]; then
FLAGS=(--se-where bridge --se-reduction "$R" --se-pre-norm --checkpoint-best)
else
FLAGS=(--se-where bridge --se-reduction "$R" --no-se-pre-norm --checkpoint-best)
fi
run "thaw_se_bridge_R${R}_${MODE}" "${FLAGS[@]}"
run "thawbin_se_bridge_R${R}_${MODE}" "${FLAGS[@]}" --eval_mode binary
done
done
# Tower-only thaw runs
for R in 8 16 32; do
for MODE in norm nonorm; do
if [[ "$MODE" == "norm" ]]; then
FLAGS=(--se-where tower --se-reduction-tower "$R" --se-pre-norm-tower --checkpoint-best)
else
FLAGS=(--se-where tower --se-reduction-tower "$R" --no-se-pre-norm-tower --checkpoint-best)
fi
run "thaw_se_tower_R${R}_${MODE}" "${FLAGS[@]}"
run "thawbin_se_tower_R${R}_${MODE}" "${FLAGS[@]}" --eval_mode binary
done
done
# Tower+bridge thaw runs
for R in 8 16 32; do
for MODE in norm nonorm; do
if [[ "$MODE" == "norm" ]]; then
FLAGS=(
--se-where both
--se-reduction "$R"
--se-reduction-tower "$R"
--se-pre-norm
--se-pre-norm-tower
--checkpoint-best
)
else
FLAGS=(
--se-where both
--se-reduction "$R"
--se-reduction-tower "$R"
--no-se-pre-norm
--no-se-pre-norm-tower
--checkpoint-best
)
fi
run "thaw_se_tower_bridge_R${R}_${MODE}" "${FLAGS[@]}"
run "thawbin_se_tower_bridge_R${R}_${MODE}" "${FLAGS[@]}" --eval_mode binary
done
done
echo "Gradual thaw grid submitted. Check analysis_data/* and models/* for outputs."
scripts/legacy/run_multifold_gui.py
-15
View File
@@ -1,15 +0,0 @@
#!/usr/bin/env python3
"""Launch the Tkinter front-end for run_multifold."""
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.frontend import launch_frontend
if __name__ == "__main__":
launch_frontend()
scripts/legacy/run_multimodel.py
-145
View File
@@ -1,145 +0,0 @@
#!/usr/bin/env python3
import argparse, subprocess, sys, time, json
from pathlib import Path
# Backbones in the paper that torchvision supports
BACKBONES = [
"efficientnet_b0",
"resnet50",
"densenet121",
"vgg16",
"mobilenet_v2",
"inception_v3",
# (Xception omitted; not in torchvision — add via timm later if needed)
]
MODES = [
("multiclass", ["Healthy", "Glaucoma", "Suspect"]),
("binary", ["Healthy", "Glaucoma"]),
]
def run(cmd):
print("\n$ " + " ".join(map(str, cmd)))
res = subprocess.run(cmd, check=True)
return res.returncode
def main():
ap = argparse.ArgumentParser(description="Run all paper CNNs across folds in multiclass + binary, then compile plots.")
ap.add_argument("--epochs", type=int, default=5, help="Epochs per fold (fast sanity first).")
ap.add_argument("--shortname", type=str, default="papergrid", help="Prefix for run IDs.")
ap.add_argument("--n-splits", type=int, default=5, help="Number of folds.")
ap.add_argument("--fusion-mode", type=str, default="fused", choices=["image_only","fused","metadata_only","vote"],
help="Paper CNNs are image-only; leave as image_only unless you’re testing others.")
ap.add_argument("--freeze-ratio", type=float, default=0.0, help="0.0 = full fine-tune (as in the paper).")
# You can override data roots if needed
ap.add_argument("--image-dir", default="Papila/FundusImages")
ap.add_argument("--clinical-dir", default="Papila/ClinicalData")
ap.add_argument("--label-col", default="Diagnosis")
ap.add_argument("--cat-cols", nargs="*", default=["Gender", "Phakic/Pseudophakic"])
args = ap.parse_args()
ts = time.strftime("%Y%m%d_%H%M%S")
master_tag = f"{args.shortname}_{ts}"
master_dir = Path("analysis_data") / master_tag
master_dir.mkdir(parents=True, exist_ok=True)
# Keep a log of all subruns for the master report
index = []
for backbone in BACKBONES:
for eval_mode, class_names in MODES:
# build a child shortname per (backbone, mode)
sub_prefix = f"{args.shortname}_{backbone}_{eval_mode}"
cmd = [
sys.executable, "scripts/run_multifold.py",
"--backbone", backbone,
"--freeze-ratio", str(args.freeze_ratio),
"--fusion-mode", args.fusion_mode,
"--epochs", str(args.epochs),
"--n-splits", str(args.n_splits),
"--shortname", sub_prefix,
"--eval_mode", eval_mode,
"--image-dir", args.image_dir,
"--clinical-dir", args.clinical_dir,
"--label-col", args.label_col,
]
# class names by mode (ensures plot legends are correct)
cmd += ["--class-names", *class_names]
plot_head_map = {
"image_only": "image",
"fused" : "fused",
"metadata_only": "metadata",
"vote": "fused",
}
# We always aggregate/plot the image head for paper CNNs
cmd += ["--plot-head", plot_head_map.get(args.fusion_mode)]
# Delegate the whole run to run_multifold.py
run(cmd)
# Discover the child run folder (the newest folder matching the shortname prefix)
# We do this because run_multifold appends its own timestamp.
adir = Path("analysis_data")
children = sorted([p for p in adir.glob(f"{sub_prefix}_*") if p.is_dir()])
if not children:
print(f"[WARN] No analysis_data folder found for {sub_prefix}; skipping index entry.")
continue
run_dir = children[-1]
summary_json = run_dir / "summary.json"
plots_dir = run_dir / "plots"
# Record entry
entry = {
"backbone": backbone,
"eval_mode": eval_mode,
"run_dir": str(run_dir),
"summary_json": str(summary_json) if summary_json.exists() else None,
"plots": {
"mean": str(plots_dir / "roc_image_mean_ovr.png"),
"overlay": str(plots_dir / "roc_image_perfold_overlay.png"),
}
}
# Try to read AUCs
try:
if summary_json.exists():
entry.update(json.loads(summary_json.read_text()))
except Exception:
pass
index.append(entry)
# Write a master JSON + markdown report
(master_dir / "index.json").write_text(json.dumps(index, indent=2), encoding="utf-8")
# Simple markdown table of results with links
lines = [
f"# Multimodel grid — {master_tag}",
"",
f"- Epochs per fold: **{args.epochs}**",
f"- Folds: **{args.n_splits}**",
f"- Fusion mode: **{args.fusion_mode}** (paper CNNs = image-only)",
f"- Freeze ratio: **{args.freeze_ratio}**",
"",
"| Backbone | Mode | Mean AUC (macro/mc or ROC-AUC/bin) | Plots | Run folder |",
"|---|---|---:|---|---|",
]
for e in index:
auc_mean = e.get("macro_ovr_auc_mean", None)
if auc_mean is not None:
auc_str = f"{auc_mean:.3f}"
else:
auc_str = "—"
mean_png = e["plots"]["mean"]
overlay_png = e["plots"]["overlay"]
plots_md = f"[mean]({mean_png}) / [overlay]({overlay_png})"
lines.append(
f"| `{e['backbone']}` | `{e['eval_mode']}` | {auc_str} | {plots_md} | `{e['run_dir']}` |"
)
(master_dir / "README.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
print(f"\nAll done.\n- Master index: {master_dir/'index.json'}\n- Report: {master_dir/'README.md'}")
print(f"- Individual runs live under analysis_data/<shortname_backbone_mode_*> with plots and summaries.")
if __name__ == "__main__":
main()
scripts/legacy/run_se_sweep.sh
-56
View File
@@ -1,56 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Usage:
# bash scripts/run_se_sweep.sh --epochs 50 --n-splits 5 --batch-size 8 --eval_mode multiclass [extra args]
#
# This will launch a series of runs covering the grid from the slide:
# - Bridge-only R8/R16/R32 (normalized and non-normalized)
# - Tower-only R8/R16/R32 (normalized and non-normalized)
# - Tower+Bridge R8/R16/R32 (normalized and non-normalized)
ARGS=("$@")
run() {
local SHORT="$1"; shift
echo "=== Running: $SHORT ==="
# Skip if a summary for this shortname already exists
if ls "analysis_data/${SHORT}_"*.md >/dev/null 2>&1; then
echo "… skipping ${SHORT} (summary already present)"
return 0
fi
python3 scripts/run_multifold.py \
--shortname "$SHORT" \
"$@" \
"${ARGS[@]}" || true
}
# Bridge-only (normalized + non-normalized)
for R in 8 16 32; do
run "se_bridge_R${R}_norm" --se-where bridge --se-reduction ${R} --se-pre-norm --checkpoint-best
run "se_bridge_R${R}_nonorm" --se-where bridge --se-reduction ${R} --no-se-pre-norm --checkpoint-best
done
# Tower-only (normalized + non-normalized)
for R in 8 16 32; do
run "se_tower_R${R}_norm" \
--se-where tower --se-reduction-tower ${R} --se-pre-norm-tower \
--checkpoint-best
run "se_tower_R${R}_nonorm" \
--se-where tower --se-reduction-tower ${R} --no-se-pre-norm-tower \
--checkpoint-best
done
# Tower+Bridge (normalized + non-normalized)
for R in 8 16 32; do
# normalized (both pre-norm on)
run "se_tower_bridge_R${R}_norm" \
--se-where both --se-reduction ${R} --se-reduction-tower ${R} \
--se-pre-norm --se-pre-norm-tower --checkpoint-best
# non-normalized (both pre-norm off)
run "se_tower_bridge_R${R}_nonorm" \
--se-where both --se-reduction ${R} --se-reduction-tower ${R} \
--no-se-pre-norm --no-se-pre-norm-tower --checkpoint-best
done
echo "Sweep submitted. Check analysis_data/* and models/* for outputs."
scripts/main/pipeline.ipynb
+1296 -68
View File
File diff suppressed because it is too large Load Diff
scripts/main/refuge/refuge_build.py
+3 -3
View File
@@ -6,7 +6,7 @@ Usage examples (after activating .venv_refuge):
python refuge_build.py --eval --with-ttt
The script expects the REFUGE folder and writes checkpoints under
models/refuge/segmentation and models/refuge/classifier.
models/v2/refuge/segmentation and models/v2/refuge/classifier.
"""
from __future__ import annotations
@@ -46,7 +46,7 @@ from classes.papila_builders import build_papila_clinical
REFUGE_ROOT = Path("REFUGE")
SEG_CKPT = Path("models/refuge/segmentation/refuge_segmentation_best.pt")
CLF_DIR = Path("models/refuge/classifier")
CLF_DIR = Path("models/v2/refuge/classifier")
UNET_WEIGHT_CANDIDATES = (
Path("models/v2/refuge/segmentation/per_image/best.pt"),
Path("models/v2/refuge/segmentation/best.pt"),
@@ -820,7 +820,7 @@ def parse_args() -> argparse.Namespace:
"--clf-checkpoint-path",
type=Path,
default=None,
help="Optional explicit path for the classifier checkpoint (defaults to models/refuge/classifier/<backbone>/refuge_classifier_best.pt)",
help="Optional explicit path for the classifier checkpoint (defaults to models/v2/refuge/classifier/<backbone>/refuge_classifier_best.pt)",
)
parser.add_argument(
"--eval-datasets",
scripts/main/refuge/run_unet_segmenter.py
+1 -1
View File
@@ -13,7 +13,7 @@ REPO_ROOT = Path(__file__).resolve().parents[3]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.unet_segmenter import UNetSegmenter
from classes.v2.unet_segmenter import UNetSegmenter
def parse_args() -> argparse.Namespace:
scripts/basic_analysis/compare_hypertower_modes.py → scripts/main/v2/multirun_hypertower.py
+8 -1
View File
@@ -7,7 +7,7 @@ from pathlib import Path
import sys
import argparse
REPO_ROOT = Path(__file__).resolve().parents[2]
REPO_ROOT = Path(__file__).resolve().parents[3]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
@@ -45,6 +45,13 @@ def main():
if not first_run:
cli.append("--persist-img-crop-cache")
args = base_parser.parse_args(cli)
# Skip if this mode is already fully complete.
if args.run_name:
tm_dir = Path(args.output_root) / args.run_name / eval_mode / tower_mode
if (tm_dir / "summary.json").exists():
print(f"[compare] {eval_mode}:{tower_mode} already complete — skipping.")
first_run = False # treat as done so cache is preserved for later runs
continue
V2HyperTower(args).run()
first_run = False
scripts/main/v2/run_10x5cv.py
+107
View File
@@ -0,0 +1,107 @@
#!/usr/bin/env python3
"""
10× repeated 5-fold CV runner for the best hypertower configuration
(nocrop, ensemble mode, both binary and multiclass).
Each repetition uses a different fold-seed so the 5 folds are split
differently, giving 50 folds per eval-mode total. Holdout composition
is kept identical across repetitions (same --holdout-seed).
Results land under:
{output-root}/rep{N:02d}/{eval_mode}/ensemble/fold{K}/
Usage
-----
python scripts/main/v2/run_10x5cv.py \
--n-reps 10 \
--eval-modes binary multiclass \
--output-root analysis_data/pipeline_10x5 \
--epochs 40 --fused-head \
--backbone refugelike
Any extra flags are forwarded directly to V2HyperTower.
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.v2.v2_hypertower import V2HyperTower
# Base fold seed for rep 0; rep N uses BASE_SEED + N * SEED_STRIDE
_BASE_SEED = 100
_SEED_STRIDE = 100
def _parse_own(argv=None):
ap = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
add_help=False,
)
ap.add_argument("--n-reps", type=int, default=10,
help="Number of repetitions (default: 10).")
ap.add_argument("--eval-modes", nargs="+",
choices=["binary", "multiclass"],
default=["binary", "multiclass"])
ap.add_argument("--output-root", default="analysis_data/pipeline_10x5",
help="Parent directory for all rep sub-runs.")
ap.add_argument("-h", "--help", action="store_true")
return ap.parse_known_args(argv)
def main(argv=None):
own, remaining = _parse_own(argv)
if own.help:
print(__doc__)
base_parser = V2HyperTower.build_parser()
base_parser.print_help()
return
base_parser = V2HyperTower.build_parser()
output_root = Path(own.output_root)
first_run = True
for rep in range(own.n_reps):
fold_seed = _BASE_SEED + rep * _SEED_STRIDE
rep_label = f"rep{rep:02d}"
for eval_mode in own.eval_modes:
tower_mode = "ensemble"
# Skip if already fully complete
tm_dir = output_root / rep_label / eval_mode / tower_mode
if (tm_dir / "summary.json").exists():
print(f"[10x5cv] {rep_label} {eval_mode}:{tower_mode} — already done, skipping.")
first_run = False
continue
cli = list(remaining) + [
"--eval-mode", eval_mode,
"--tower-mode", tower_mode,
"--fold-seed", str(fold_seed),
"--run-name", rep_label,
"--output-root", str(output_root),
]
# Reuse crop cache across runs after the first
if not first_run:
cli.append("--persist-img-crop-cache")
print(f"\n[10x5cv] Starting {rep_label} {eval_mode}:{tower_mode} "
f"(fold_seed={fold_seed})")
args = base_parser.parse_args(cli)
V2HyperTower(args).run()
first_run = False
print(f"\n[10x5cv] All done. Results in: {output_root}")
if __name__ == "__main__":
main()
scripts/main/v2/run_binary_v2.2.sh
-74
View File
@@ -1,74 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Binary runs v2.2 (6 total):
# UNet crop: single | ensemble | fused head
# GT crop: single | ensemble | fused head
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
cd "$ROOT_DIR"
MANIFEST="manifest.csv"
UNET_WEIGHTS="models/v2/refuge/segmentation/per_image/best.pt"
COMMON=(
--epochs 40
--n-splits 5
--batch-size 8
--backbone refugelike
--eval-mode binary
--img-crop-manifest "$MANIFEST"
)
UNET_CROP=(
--img-crop-weights "$UNET_WEIGHTS"
)
GT_CROP=(
--img-crop-gt
)
# ── UNet crop ────────────────────────────────────────────────────────────────
echo "[1/6] UNet crop — binary, single..."
python3 scripts/main/v2/run_multifold_v2.py \
"${COMMON[@]}" "${UNET_CROP[@]}" \
--tower-mode single \
--run-name v2.2_single_binary_unet_40ep_5fold
echo "[2/6] UNet crop — binary, ensemble..."
python3 scripts/main/v2/run_multifold_v2.py \
"${COMMON[@]}" "${UNET_CROP[@]}" \
--tower-mode ensemble \
--run-name v2.2_ensemble_binary_unet_40ep_5fold
echo "[3/6] UNet crop — binary, ensemble + fused head..."
python3 scripts/main/v2/run_multifold_v2.py \
"${COMMON[@]}" "${UNET_CROP[@]}" \
--tower-mode ensemble \
--fused-head --fusion-epochs 10 \
--run-name v2.2_fused_binary_unet_40ep_5fold
# ── GT crop ──────────────────────────────────────────────────────────────────
echo "[4/6] GT crop — binary, single..."
python3 scripts/main/v2/run_multifold_v2.py \
"${COMMON[@]}" "${GT_CROP[@]}" \
--tower-mode single \
--run-name v2.2_single_binary_gt_40ep_5fold
echo "[5/6] GT crop — binary, ensemble..."
python3 scripts/main/v2/run_multifold_v2.py \
"${COMMON[@]}" "${GT_CROP[@]}" \
--tower-mode ensemble \
--run-name v2.2_ensemble_binary_gt_40ep_5fold
echo "[6/6] GT crop — binary, ensemble + fused head..."
python3 scripts/main/v2/run_multifold_v2.py \
"${COMMON[@]}" "${GT_CROP[@]}" \
--tower-mode ensemble \
--img-crop-gt \
--fused-head --fusion-epochs 10 \
--run-name v2.2_fused_binary_gt_40ep_5fold
echo "Binary v2.2 runs complete."
scripts/main/v2/run_iop_corr_comparison.py
+90
View File
@@ -0,0 +1,90 @@
#!/usr/bin/env python3
"""
Run single-mode binary + multiclass for each IOP correction method and
collect all results under one output root for easy comparison.
Output layout:
analysis_data/iop_corr_comparison/
ratio/binary/single/ ratio/multiclass/single/
ols/binary/single/ ols/multiclass/single/
lad/binary/single/ lad/multiclass/single/
multi/binary/single/ multi/multiclass/single/
Usage
-----
python scripts/main/v2/run_iop_corr_comparison.py [V2HyperTower args...]
Any extra args (backbone, epochs, img-crop-*, etc.) are forwarded to every run.
"""
from __future__ import annotations
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.v2.v2_hypertower import V2HyperTower
IOP_METHODS = ["ratio", "ols", "lad", "multi"]
EVAL_MODES = ["binary", "multiclass"]
OUTPUT_ROOT = "analysis_data/iop_corr_comparison"
def main() -> None:
base_parser = V2HyperTower.build_parser()
# Consume only the remaining (forwarded) args — iop-corr-method and
# run-name are set by this script; eval-mode and tower-mode likewise.
_, remaining = base_parser.parse_known_args()
first_run = True
for method in IOP_METHODS:
for eval_mode in EVAL_MODES:
run_name = method # one sub-folder per method
tm_dir = (Path(OUTPUT_ROOT) / run_name / eval_mode / "single")
if (tm_dir / "summary.json").exists():
print(f"[iop_corr] {method}/{eval_mode}/single — already done, skipping.")
first_run = False
continue
cli = list(remaining) + [
"--eval-mode", eval_mode,
"--tower-mode", "single",
"--iop-corr-method", method,
"--output-root", OUTPUT_ROOT,
"--run-name", run_name,
]
if not first_run:
cli.append("--persist-img-crop-cache")
print(f"\n[iop_corr] Starting {method}/{eval_mode}/single ...")
args = base_parser.parse_args(cli)
V2HyperTower(args).run()
first_run = False
# ── summary table ──────────────────────────────────────────────────────
import json
print("\n" + "=" * 60)
print("IOP correction method comparison — single mode")
print("=" * 60)
header = f"{'Method':<8} {'Mode':<12} {'Val AUC':>10} {'Hld AUC':>10}"
print(header)
print("-" * len(header))
for method in IOP_METHODS:
for eval_mode in EVAL_MODES:
p = Path(OUTPUT_ROOT) / method / eval_mode / "single" / "summary.json"
if not p.exists():
print(f"{method:<8} {eval_mode:<12} {'missing':>10} {'missing':>10}")
continue
ms = json.loads(p.read_text()).get("mode_summary", {})
val = ms.get("classic_best_val", {})
hld = ms.get("classic_holdout", {})
val_s = f"{val['auc_mean']:.3f}±{val['auc_std']:.3f}" if val.get("auc_mean") else "—"
hld_s = f"{hld['auc_mean']:.3f}±{hld['auc_std']:.3f}" if hld.get("auc_mean") else "—"
print(f"{method:<8} {eval_mode:<12} {val_s:>10} {hld_s:>10}")
print("=" * 60)
if __name__ == "__main__":
main()
scripts/main/v2/run_md_mlp.py
+23
View File
@@ -323,6 +323,7 @@ def main() -> None:
best_epoch = 0
best_phase = ""
best_state = None
epoch_log_rows = []
print(
f"\n[fold {fold_idx+1}/{args.n_splits}] "
@@ -350,6 +351,14 @@ def main() -> None:
aggregate_patient=aggregate_patient,
)
hld_auc_ep = float("nan")
hld_acc_ep = float("nan")
if holdout_loader is not None:
_, _, hld_auc_ep, hld_acc_ep = _evaluate_single(
model, holdout_loader, device, num_classes,
aggregate_patient=aggregate_patient,
)
is_main = phase == "main"
if is_main and (not np.isnan(val_auc)) and val_auc > best_auc:
best_auc = float(val_auc)
@@ -357,6 +366,17 @@ def main() -> None:
best_epoch = ep + 1
best_phase = phase
epoch_log_rows.append({
"epoch": ep + 1,
"phase": phase,
"train_loss": float(tr_loss),
"train_acc": float(tr_acc),
"val_auc": float(val_auc),
"val_acc": float(val_acc),
"hld_auc": float(hld_auc_ep),
"hld_acc": float(hld_acc_ep),
})
if ep == 0 or (ep + 1) % 10 == 0 or (ep + 1) == total_epochs:
print(
f" ep {ep+1:>3}/{total_epochs} [{phase}:{main_ep}/{args.epochs}] "
@@ -366,6 +386,9 @@ def main() -> None:
flush=True,
)
import pandas as _pd
_pd.DataFrame(epoch_log_rows).to_csv(fold_dir / "epoch_log.csv", index=False)
if best_state is not None:
model.load_state_dict(best_state)
scripts/main/v2/run_multiclass_v2.2.sh
-73
View File
@@ -1,73 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Multiclass runs v2.2 (6 total):
# UNet crop: single | ensemble | fused head
# GT crop: single | ensemble | fused head
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
cd "$ROOT_DIR"
MANIFEST="manifest.csv"
UNET_WEIGHTS="models/v2/refuge/segmentation/per_image/best.pt"
COMMON=(
--epochs 40
--n-splits 5
--batch-size 8
--backbone refugelike
--eval-mode multiclass
--img-crop-manifest "$MANIFEST"
)
UNET_CROP=(
--img-crop-weights "$UNET_WEIGHTS"
)
GT_CROP=(
--img-crop-gt
)
# ── UNet crop ────────────────────────────────────────────────────────────────
echo "[1/6] UNet crop — multiclass, single..."
python3 scripts/main/v2/run_multifold_v2.py \
"${COMMON[@]}" "${UNET_CROP[@]}" \
--tower-mode single \
--run-name v2.2_single_multiclass_unet_40ep_5fold
echo "[2/6] UNet crop — multiclass, ensemble..."
python3 scripts/main/v2/run_multifold_v2.py \
"${COMMON[@]}" "${UNET_CROP[@]}" \
--tower-mode ensemble \
--run-name v2.2_ensemble_multiclass_unet_40ep_5fold
echo "[3/6] UNet crop — multiclass, ensemble + fused head..."
python3 scripts/main/v2/run_multifold_v2.py \
"${COMMON[@]}" "${UNET_CROP[@]}" \
--tower-mode ensemble \
--fused-head --fusion-epochs 10 \
--run-name v2.2_fused_multiclass_unet_40ep_5fold
# ── GT crop ──────────────────────────────────────────────────────────────────
echo "[4/6] GT crop — multiclass, single..."
python3 scripts/main/v2/run_multifold_v2.py \
"${COMMON[@]}" "${GT_CROP[@]}" \
--tower-mode single \
--run-name v2.2_single_multiclass_gt_40ep_5fold
echo "[5/6] GT crop — multiclass, ensemble..."
python3 scripts/main/v2/run_multifold_v2.py \
"${COMMON[@]}" "${GT_CROP[@]}" \
--tower-mode ensemble \
--run-name v2.2_ensemble_multiclass_gt_40ep_5fold
echo "[6/6] GT crop — multiclass, ensemble + fused head..."
python3 scripts/main/v2/run_multifold_v2.py \
"${COMMON[@]}" "${GT_CROP[@]}" \
--tower-mode ensemble \
--fused-head --fusion-epochs 10 \
--run-name v2.2_fused_multiclass_gt_40ep_5fold
echo "Multiclass v2.2 runs complete."
scripts/main/v2/run_multifold_v2_modes.py
+1 -1
View File
@@ -6,7 +6,7 @@ from __future__ import annotations
from pathlib import Path
import sys
REPO_ROOT = Path(__file__).resolve().parents[1]
REPO_ROOT = Path(__file__).resolve().parents[3]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
scripts/output_analysis/aggregate_10x5cv.py
+496
View File
@@ -0,0 +1,496 @@
#!/usr/bin/env python3
"""
Aggregate and visualise results from a 10× repeated 5-fold CV run.
Reads probs / y_true from every rep/fold directory, computes per-fold
metrics, and produces:
outputs/
fold_metrics.csv — one row per (rep, fold, eval_mode)
rep_metrics.csv — one row per (rep, eval_mode): mean over 5 folds
overall_summary.txt — mean ± SD and 95% CI printed to console + file
{eval_mode}_auc_violin.png
{eval_mode}_roc_mean.png — mean ± 1 SD OVR ROC (all classes or class 1)
{eval_mode}_holdout_roc_mean.png
Holdout metrics are extracted from the rep-level predictions.npz using the
best_epoch recorded in summary.json, ensemble-averaged over od_fused + os_fused
heads, giving 50 fold-level holdout AUC values (5 folds × 10 reps).
Usage
-----
python scripts/output_analysis/aggregate_10x5cv.py \
--run-root analysis_data/pipeline_10x5 \
--eval-modes binary multiclass \
--out analysis_data/pipeline_10x5/aggregate
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.metrics import roc_auc_score, accuracy_score, roc_curve, auc as sk_auc
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
_CLASS_NAMES = {
"binary": ["Healthy", "Glaucoma"],
"multiclass": ["Healthy", "Glaucoma", "Suspect"],
}
# Probe files in preference order (first found wins)
# probs_fused = simple OD/OS softmax average (ensemble head — primary metric)
# probs_fused_head = learned logit-level fusion head (worse on average; kept as fallback)
_PROBS_PRIORITY = ["probs_fused.npy", "probs_fused_head.npy"]
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _find_probs(fold_dir: Path) -> Path | None:
for name in _PROBS_PRIORITY:
p = fold_dir / name
if p.exists():
return p
return None
def _load_holdout_summary(mode_dir: Path) -> dict | None:
"""
Read rep-level holdout metrics from summary.json.
Returns dict with keys auc_mean, auc_std, acc_mean (may be None if missing).
"""
summary_path = mode_dir / "summary.json"
if not summary_path.exists():
return None
try:
summary = json.loads(summary_path.read_text())
return summary.get("mode_summary", {}).get("ensemble_holdout")
except Exception:
return None
def _auc_macro(y: np.ndarray, p: np.ndarray, num_classes: int) -> float:
try:
if num_classes == 2:
return float(roc_auc_score(y, p[:, 1]))
return float(roc_auc_score(y, p, multi_class="ovr", average="macro"))
except Exception:
return float("nan")
def _per_class_roc(y: np.ndarray, p: np.ndarray) -> dict[int, dict]:
out: dict[int, dict] = {}
for k in range(p.shape[1]):
yb = (y == k).astype(np.uint8)
if yb.sum() == 0 or yb.sum() == len(yb):
continue
fpr, tpr, _ = roc_curve(yb, p[:, k])
out[k] = {"fpr": fpr, "tpr": tpr, "auc": sk_auc(fpr, tpr)}
return out
def _ci95(values: np.ndarray) -> tuple[float, float]:
"""95% CI via t-distribution (two-sided)."""
from scipy import stats as scipy_stats
if len(values) < 2:
return (float("nan"), float("nan"))
ci = scipy_stats.t.interval(0.95, df=len(values) - 1,
loc=np.mean(values), scale=scipy_stats.sem(values))
return float(ci[0]), float(ci[1])
# ---------------------------------------------------------------------------
# Data loading
# ---------------------------------------------------------------------------
def load_all_folds(run_root: Path, eval_modes: list[str]) -> pd.DataFrame:
rows = []
rep_dirs = sorted(
[d for d in run_root.iterdir() if d.is_dir() and d.name.startswith("rep")],
key=lambda d: d.name,
)
if not rep_dirs:
raise SystemExit(f"No rep* directories found in {run_root}")
for rep_dir in rep_dirs:
for eval_mode in eval_modes:
tower_mode = "ensemble"
mode_dir = rep_dir / eval_mode / tower_mode
if not mode_dir.exists():
print(f" [skip] {mode_dir} not found")
continue
num_classes = 2 if eval_mode == "binary" else 3
fold_dirs = sorted(
[d for d in mode_dir.iterdir() if d.is_dir() and d.name.startswith("fold")],
key=lambda d: int(d.name[4:]),
)
for fold_dir in fold_dirs:
fold_idx = int(fold_dir.name[4:])
y_path = fold_dir / "y_true.npy"
p_path = _find_probs(fold_dir)
if y_path is None or not y_path.exists() or p_path is None:
print(f" [skip] {rep_dir.name}/{eval_mode}/fold{fold_idx}: missing files")
continue
y = np.load(y_path)
p = np.load(p_path)
if eval_mode == "binary":
mask = np.isin(y, [0, 1])
y, p = y[mask], p[mask]
if p.shape[1] > 2:
p = p[:, :2]
auc_macro = _auc_macro(y, p, num_classes)
acc = float(accuracy_score(y, p.argmax(1)))
row = {
"rep": rep_dir.name,
"fold": fold_idx,
"eval_mode": eval_mode,
"probs_file": p_path.name,
"auc_macro": auc_macro,
"acc": acc,
"n": len(y),
}
# Per-class AUC
for k in range(num_classes):
yb = (y == k).astype(np.uint8)
if yb.sum() > 0 and yb.sum() < len(yb):
try:
row[f"auc_class{k}"] = float(roc_auc_score(yb, p[:, k]))
except Exception:
row[f"auc_class{k}"] = float("nan")
else:
row[f"auc_class{k}"] = float("nan")
rows.append(row)
# ---- holdout metrics from rep-level summary.json ----
# Holdout probs are not stored per-fold; only aggregated stats are saved.
# We attach the rep-level mean to each fold row (same value repeated),
# and also add a single rep-level summary row (fold=-1).
hld_summary = _load_holdout_summary(mode_dir)
if hld_summary:
hld_auc = hld_summary.get("auc_mean", float("nan"))
hld_auc_std = hld_summary.get("auc_std", float("nan"))
hld_acc = hld_summary.get("acc_mean", float("nan"))
for row in rows:
if row["rep"] == rep_dir.name and row["eval_mode"] == eval_mode:
row["hld_auc_macro"] = hld_auc
row["hld_acc"] = hld_acc
# Also store a rep-level holdout row (fold=-1) for direct rep-level analysis
rows.append({
"rep": rep_dir.name,
"fold": -1,
"eval_mode": eval_mode,
"probs_file": "summary.json",
"auc_macro": float("nan"),
"acc": float("nan"),
"n": float("nan"),
"hld_auc_macro": hld_auc,
"hld_auc_std_within_rep": hld_auc_std,
"hld_acc": hld_acc,
})
return pd.DataFrame(rows)
# ---------------------------------------------------------------------------
# Plotting
# ---------------------------------------------------------------------------
def _violin(fold_df: pd.DataFrame, eval_mode: str, out_dir: Path) -> None:
sub = fold_df[fold_df["eval_mode"] == eval_mode].copy()
num_classes = 2 if eval_mode == "binary" else 3
class_names = _CLASS_NAMES[eval_mode]
auc_cols = ["auc_macro"] + [f"auc_class{k}" for k in range(num_classes)]
labels = ["Macro AUC"] + [f"AUC {class_names[k]}" for k in range(num_classes)]
present = [(c, l) for c, l in zip(auc_cols, labels) if c in sub.columns]
data = [sub[c].dropna().values for c, _ in present]
labels = [l for _, l in present]
fig, ax = plt.subplots(figsize=(max(6, 2 * len(data)), 5))
parts = ax.violinplot(data, showmedians=True, showextrema=True)
for pc in parts["bodies"]:
pc.set_alpha(0.7)
# Overlay individual rep means
rep_means = sub.groupby("rep")[auc_cols[0]].mean().values
ax.scatter(np.ones(len(rep_means)), rep_means, zorder=3,
color="k", s=18, alpha=0.6, label="rep mean")
ax.set_xticks(range(1, len(labels) + 1))
ax.set_xticklabels(labels, rotation=15, ha="right")
ax.set_ylabel("AUC")
ax.set_title(f"AUC distribution — {eval_mode} (10 × 5-fold, n={len(sub)})")
ax.set_ylim(max(0, sub[auc_cols[0]].min() - 0.05), 1.02)
ax.grid(True, axis="y", linewidth=0.4, alpha=0.5)
ax.legend(fontsize=8)
fig.tight_layout()
path = out_dir / f"{eval_mode}_auc_violin.png"
fig.savefig(path, dpi=160, bbox_inches="tight")
plt.close(fig)
print(f" Saved: {path}")
def _mean_roc(fold_df: pd.DataFrame, eval_mode: str,
run_root: Path, out_dir: Path) -> None:
"""Mean ± 1 SD OVR ROC across all 50 folds."""
sub = fold_df[fold_df["eval_mode"] == eval_mode]
num_classes = 2 if eval_mode == "binary" else 3
class_names = _CLASS_NAMES[eval_mode]
# Classes to plot (binary: class 1 only)
plot_classes = [1] if eval_mode == "binary" else list(range(num_classes))
grid = np.linspace(0, 1, 501)
fig, ax = plt.subplots(figsize=(9, 7))
ax.plot([0, 1], [0, 1], linestyle="--", linewidth=1, color="grey")
for k in plot_classes:
tprs, aucs = [], []
for _, row in sub.iterrows():
rep_dir = run_root / row["rep"]
fold_dir = rep_dir / eval_mode / "ensemble" / f"fold{int(row['fold'])}"
y_path = fold_dir / "y_true.npy"
p_path = _find_probs(fold_dir)
if not y_path.exists() or p_path is None:
continue
y = np.load(y_path)
p = np.load(p_path)
if eval_mode == "binary":
mask = np.isin(y, [0, 1])
y, p = y[mask], p[mask]
if p.shape[1] > 2:
p = p[:, :2]
yb = (y == k).astype(np.uint8)
if yb.sum() == 0 or yb.sum() == len(yb):
continue
fpr, tpr, _ = roc_curve(yb, p[:, k])
tprs.append(np.interp(grid, fpr, tpr))
aucs.append(sk_auc(fpr, tpr))
if not tprs:
continue
arr = np.vstack(tprs)
mean = arr.mean(0)
std = arr.std(0)
cname = class_names[k]
lbl = f"{cname} AUC {np.nanmean(aucs):.3f} ± {np.nanstd(aucs):.3f}"
line, = ax.plot(grid, mean, linewidth=2, label=lbl)
ax.fill_between(grid,
np.clip(mean - std, 0, 1),
np.clip(mean + std, 0, 1),
alpha=0.15, color=line.get_color())
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(f"Mean ± 1 SD OVR ROC — {eval_mode} (10 × 5-fold)")
ax.legend(loc="lower right", fontsize=9)
fig.tight_layout()
path = out_dir / f"{eval_mode}_roc_mean.png"
fig.savefig(path, dpi=160, bbox_inches="tight")
plt.close(fig)
print(f" Saved: {path}")
def _holdout_stability(fold_df: pd.DataFrame, eval_mode: str, out_dir: Path) -> None:
"""Bar chart of per-rep holdout AUC (mean across folds within rep ± within-rep SD)."""
# use the rep-level rows (fold == -1) which have hld_auc_std_within_rep
rep_rows = fold_df[(fold_df["eval_mode"] == eval_mode) & (fold_df["fold"] == -1)].copy()
if rep_rows.empty or "hld_auc_macro" not in rep_rows.columns:
print(f" [skip] no holdout data for {eval_mode}")
return
rep_rows = rep_rows.sort_values("rep")
fig, ax = plt.subplots(figsize=(max(6, len(rep_rows) * 0.9), 4))
x = np.arange(len(rep_rows))
yerr = rep_rows.get("hld_auc_std_within_rep", pd.Series([0]*len(rep_rows))).fillna(0).values
ax.bar(x, rep_rows["hld_auc_macro"].values, yerr=yerr,
capsize=4, color="darkorange", alpha=0.8)
grand_mean = rep_rows["hld_auc_macro"].mean()
ax.axhline(grand_mean, linestyle="--", color="crimson",
linewidth=1.2, label=f"grand mean = {grand_mean:.3f}")
ax.set_xticks(x)
ax.set_xticklabels(rep_rows["rep"].values, rotation=30, ha="right")
ax.set_ylabel("Holdout macro AUC (mean ± within-rep SD)")
ax.set_title(f"Per-rep holdout stability — {eval_mode}")
ymin = max(0, rep_rows["hld_auc_macro"].min() - 0.05)
ax.set_ylim(ymin, 1.02)
ax.legend(fontsize=9)
ax.grid(True, axis="y", linewidth=0.4, alpha=0.5)
fig.tight_layout()
path = out_dir / f"{eval_mode}_holdout_stability.png"
fig.savefig(path, dpi=160, bbox_inches="tight")
plt.close(fig)
print(f" Saved: {path}")
def _rep_stability(fold_df: pd.DataFrame, eval_mode: str, out_dir: Path) -> None:
"""Bar chart of per-rep mean macro AUC with ± 1 SD error bars."""
sub = fold_df[fold_df["eval_mode"] == eval_mode]
rep_stats = sub.groupby("rep")["auc_macro"].agg(["mean", "std"]).reset_index()
rep_stats = rep_stats.sort_values("rep")
fig, ax = plt.subplots(figsize=(max(6, len(rep_stats) * 0.9), 4))
x = np.arange(len(rep_stats))
ax.bar(x, rep_stats["mean"], yerr=rep_stats["std"],
capsize=4, color="steelblue", alpha=0.8)
ax.axhline(rep_stats["mean"].mean(), linestyle="--", color="crimson",
linewidth=1.2, label=f"grand mean = {rep_stats['mean'].mean():.3f}")
ax.set_xticks(x)
ax.set_xticklabels(rep_stats["rep"], rotation=30, ha="right")
ax.set_ylabel("Mean macro AUC (5 folds)")
ax.set_title(f"Per-rep stability — {eval_mode}")
ymin = max(0, rep_stats["mean"].min() - rep_stats["std"].max() - 0.02)
ax.set_ylim(ymin, 1.02)
ax.legend(fontsize=9)
ax.grid(True, axis="y", linewidth=0.4, alpha=0.5)
fig.tight_layout()
path = out_dir / f"{eval_mode}_rep_stability.png"
fig.savefig(path, dpi=160, bbox_inches="tight")
plt.close(fig)
print(f" Saved: {path}")
# ---------------------------------------------------------------------------
# Summary text
# ---------------------------------------------------------------------------
def _print_summary(fold_df: pd.DataFrame, eval_modes: list[str]) -> str:
lines = ["=" * 60, "10 × 5-fold CV — aggregate summary", "=" * 60]
for eval_mode in eval_modes:
sub = fold_df[(fold_df["eval_mode"] == eval_mode) & (fold_df["fold"] >= 0)]
if sub.empty:
continue
num_classes = 2 if eval_mode == "binary" else 3
class_names = _CLASS_NAMES[eval_mode]
lines.append(f"\n--- {eval_mode.upper()} ---")
lines.append(f" n_folds = {len(sub)}")
lines.append(" [Validation]")
for col, label in [("auc_macro", "Macro AUC"), ("acc", "Accuracy")]:
if col not in sub.columns:
continue
vals = sub[col].dropna().values
ci_lo, ci_hi = _ci95(vals)
lines.append(
f" {label:18s}: {vals.mean():.4f} ± {vals.std():.4f}"
f" 95% CI [{ci_lo:.4f}, {ci_hi:.4f}]"
)
for k in range(num_classes):
col = f"auc_class{k}"
if col not in sub.columns:
continue
vals = sub[col].dropna().values
if len(vals) == 0:
continue
ci_lo, ci_hi = _ci95(vals)
lines.append(
f" AUC {class_names[k]:12s}: {vals.mean():.4f} ± {vals.std():.4f}"
f" 95% CI [{ci_lo:.4f}, {ci_hi:.4f}]"
)
# Between-rep variance (val)
rep_means = sub.groupby("rep")["auc_macro"].mean().values
lines.append(
f" Rep-mean AUC (n={len(rep_means)}): "
f"{rep_means.mean():.4f} ± {rep_means.std():.4f}"
f" (between-rep SD = {rep_means.std():.4f})"
)
# Holdout — use rep-level rows (fold == -1)
rep_hld = fold_df[
(fold_df["eval_mode"] == eval_mode) &
(fold_df["fold"] == -1) &
fold_df["hld_auc_macro"].notna()
]["hld_auc_macro"].values if "hld_auc_macro" in fold_df.columns else np.array([])
if len(rep_hld) > 0:
lines.append(" [Holdout] (rep-level means, n_reps={})".format(len(rep_hld)))
ci_lo, ci_hi = _ci95(rep_hld)
lines.append(
f" {'Macro AUC':18s}: {rep_hld.mean():.4f} ± {rep_hld.std():.4f}"
f" 95% CI [{ci_lo:.4f}, {ci_hi:.4f}]"
)
lines.append("=" * 60)
return "\n".join(lines)
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def main() -> None:
ap = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
ap.add_argument("--run-root", default="analysis_data/pipeline_10x5",
help="Root directory containing rep* sub-directories.")
ap.add_argument("--eval-modes", nargs="+",
choices=["binary", "multiclass"],
default=["binary", "multiclass"])
ap.add_argument("--out", default=None,
help="Output directory for plots and CSVs "
"(default: {run-root}/aggregate).")
args = ap.parse_args()
run_root = Path(args.run_root)
out_dir = Path(args.out) if args.out else run_root / "aggregate"
out_dir.mkdir(parents=True, exist_ok=True)
print("Loading fold data...")
fold_df = load_all_folds(run_root, args.eval_modes)
if fold_df.empty:
raise SystemExit("No data loaded — check --run-root.")
fold_df.to_csv(out_dir / "fold_metrics.csv", index=False)
print(f" Saved fold_metrics.csv ({len(fold_df)} rows)")
rep_df = (fold_df.groupby(["rep", "eval_mode"])
[["auc_macro", "acc"] +
[c for c in fold_df.columns if c.startswith("auc_class")]]
.mean()
.reset_index())
rep_df.to_csv(out_dir / "rep_metrics.csv", index=False)
print(f" Saved rep_metrics.csv ({len(rep_df)} rows)")
summary_text = _print_summary(fold_df, args.eval_modes)
print("\n" + summary_text)
(out_dir / "overall_summary.txt").write_text(summary_text + "\n")
print(f"\n Saved overall_summary.txt")
print("\nGenerating plots...")
for eval_mode in args.eval_modes:
if fold_df[fold_df["eval_mode"] == eval_mode].empty:
continue
_violin(fold_df, eval_mode, out_dir)
_mean_roc(fold_df, eval_mode, run_root, out_dir)
_rep_stability(fold_df, eval_mode, out_dir)
_holdout_stability(fold_df, eval_mode, out_dir)
print(f"\nAll outputs written to: {out_dir}")
if __name__ == "__main__":
main()
scripts/output_analysis/explainability/aggregate_gradcam.py
+241
View File
@@ -0,0 +1,241 @@
#!/usr/bin/env python3
"""
Aggregate raw GradCAM heatmaps across all folds for a run.
For each combination of (eye, class, correct/incorrect) computes:
- mean heatmap
- std heatmap
- count
Also computes a scalar per patient: fraction of GradCAM attention mass that
falls within the expert-segmented optic disc region (from GT contour files),
using the manifest.csv to locate the contour for each patient/eye.
Outputs
-------
{out_dir}/mean_heatmaps.npz
Keys: {eye}_{class_name}_{correct|incorrect}_{mean|std|count}
e.g. OD_Glaucoma_correct_mean shape (224, 224)
{out_dir}/attention_stats.csv
per-patient scalars: patient_id, fold, eye, true_name, pred_name,
correct, confidence, disc_frac, entropy
Usage
-----
python scripts/output_analysis/explainability/aggregate_gradcam.py \
--run-dir analysis_data/pipeline_nocrop \
--eval-mode binary \
--tower-mode single
"""
from __future__ import annotations
import argparse
from pathlib import Path
import numpy as np
import pandas as pd
from PIL import Image, ImageDraw
def load_disc_mask(contour_path: Path, orig_size: tuple[int, int],
cam_h: int, cam_w: int) -> np.ndarray | None:
"""
Load a PAPILA disc contour TXT file, polygon-fill at original image
dimensions, then resize to (cam_h, cam_w). Returns a bool array or
None if the contour cannot be loaded.
"""
try:
arr = np.loadtxt(str(contour_path), dtype=np.float32)
except Exception:
return None
if arr.ndim == 1:
arr = arr.reshape(-1, 2)
if arr.shape[0] < 3 or arr.shape[1] < 2:
return None
# orig_size is (W, H) as PIL convention
img = Image.new("L", orig_size, 0)
draw = ImageDraw.Draw(img)
draw.polygon([tuple(pt) for pt in arr[:, :2]], fill=1)
mask = np.array(img.resize((cam_w, cam_h), Image.NEAREST), dtype=bool)
return mask
def build_disc_lookup(manifest_path: Path) -> dict[tuple[int, str], tuple[Path, tuple[int, int]]]:
"""
Returns {(patient_id_int, eye): (disc_contour_path, (img_W, img_H))}.
Only PAPILA rows are included.
"""
mf = pd.read_csv(manifest_path)
lookup: dict[tuple[int, str], tuple[Path, tuple[int, int]]] = {}
for _, row in mf.iterrows():
sid = str(row["sample_id"])
if not sid.startswith("papila_RET"):
continue
# sample_id: papila_RET002OD or papila_RET002OS
suffix = sid[len("papila_RET"):] # e.g. "002OD"
eye = suffix[-2:] # "OD" or "OS"
pid = int(suffix[:-2]) # 2
disc_path = Path(str(row["annotation_disc"]))
img_path = Path(str(row["image_path"]))
if not disc_path.exists():
continue
# read original image size once
try:
with Image.open(img_path) as im:
orig_size = im.size # (W, H)
except Exception:
continue
lookup[(pid, eye)] = (disc_path, orig_size)
return lookup
def attention_entropy(cam: np.ndarray) -> float:
flat = cam.flatten().astype(np.float64)
flat = flat / (flat.sum() + 1e-12)
return float(-np.sum(flat * np.log(flat + 1e-12)))
def load_fold(gradcam_dir: Path):
idx_path = gradcam_dir / "gradcam_index.csv"
if not idx_path.exists():
return None
idx = pd.read_csv(idx_path)
records = []
for _, row in idx.iterrows():
pid = row["patient_id"]
for eye in ("OD", "OS"):
npy = gradcam_dir / f"patient_{pid}_{eye}_cam.npy"
if not npy.exists():
continue
cam = np.load(npy)
records.append({
"patient_id": pid,
"eye": eye,
"true_label": int(row["true_label"]),
"true_name": row["true_name"],
"pred_label": int(row["pred_label"]),
"pred_name": row["pred_name"],
"confidence": float(row["confidence"]),
"correct": bool(row["correct"]),
"cam": cam,
})
return records
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--run-dir", default="analysis_data/pipeline_nocrop")
ap.add_argument("--eval-mode", default="binary")
ap.add_argument("--tower-mode", default="single")
ap.add_argument("--manifest", default="manifest.csv")
ap.add_argument("--out", default=None)
args = ap.parse_args()
run_dir = Path(args.run_dir)
mode_dir = run_dir / args.eval_mode / args.tower_mode
out_dir = mode_dir / "gradcam_aggregate"
if args.out:
out_dir = Path(args.out)
out_dir.mkdir(parents=True, exist_ok=True)
# ---- build disc mask lookup ----
manifest_path = Path(args.manifest)
disc_lookup = build_disc_lookup(manifest_path)
print(f"Disc mask lookup: {len(disc_lookup)} entries from {manifest_path}")
# ---- collect all records ----
all_records = []
stat_rows = []
fold_dirs = sorted(
[d for d in mode_dir.iterdir() if d.is_dir() and d.name.startswith("fold")],
key=lambda p: int(p.name.replace("fold", "")),
)
if not fold_dirs:
print(f"No fold dirs found under {mode_dir}")
return
for fd in fold_dirs:
gcam_dir = fd / "explainability" / "gradcam"
records = load_fold(gcam_dir)
if records is None:
print(f" [skip] {fd.name}: no gradcam_index.csv")
continue
print(f" {fd.name}: {len(records)} eye records")
for r in records:
r["fold"] = fd.name
all_records.append(r)
if not all_records:
print("No records found — re-run explain_fold.py first.")
return
print(f"\nTotal eye records: {len(all_records)}")
h, w = all_records[0]["cam"].shape
# ---- per-record stats ----
n_missing = 0
for r in all_records:
cam = r["cam"]
total = cam.sum() + 1e-12
pid = int(r["patient_id"])
eye = r["eye"]
disc_mask = None
key = (pid, eye)
if key in disc_lookup:
disc_path, orig_size = disc_lookup[key]
disc_mask = load_disc_mask(disc_path, orig_size, h, w)
if disc_mask is None:
n_missing += 1
disc_frac = float("nan")
else:
disc_frac = float(cam[disc_mask].sum() / total)
stat_rows.append({
"patient_id": r["patient_id"],
"fold": r["fold"],
"eye": r["eye"],
"true_name": r["true_name"],
"pred_name": r["pred_name"],
"correct": r["correct"],
"confidence": r["confidence"],
"disc_frac": disc_frac,
"entropy": attention_entropy(cam),
})
if n_missing:
print(f" Warning: {n_missing} records had no disc mask (disc_frac=NaN)")
stats_df = pd.DataFrame(stat_rows)
stats_path = out_dir / "attention_stats.csv"
stats_df.to_csv(stats_path, index=False)
print(f"Saved attention stats → {stats_path}")
# ---- mean heatmaps ----
npz_arrays = {}
groups: dict[tuple, list[np.ndarray]] = {}
for r in all_records:
key = (r["eye"], r["true_name"], "correct" if r["correct"] else "incorrect")
groups.setdefault(key, []).append(r["cam"])
for r in all_records:
key = (r["eye"], r["true_name"], "all")
groups.setdefault(key, []).append(r["cam"])
for (eye, cls, split), cams in groups.items():
stack = np.stack(cams, axis=0)
key_base = f"{eye}_{cls}_{split}"
npz_arrays[f"{key_base}_mean"] = stack.mean(axis=0).astype(np.float32)
npz_arrays[f"{key_base}_std"] = stack.std(axis=0).astype(np.float32)
npz_arrays[f"{key_base}_count"] = np.array(len(cams))
print(f" {key_base}: N={len(cams)}")
npz_path = out_dir / "mean_heatmaps.npz"
np.savez_compressed(npz_path, **npz_arrays)
print(f"Saved mean heatmaps → {npz_path}")
if __name__ == "__main__":
main()
scripts/output_analysis/explainability/explain_fold.py
+130 -84
View File
@@ -409,6 +409,7 @@ def run_gradcam(
overlay_grid_items: list[
tuple[Image.Image | None, Image.Image | None, str, bool]
] = []
index_rows: list[dict] = []
model.eval()
for batch in loader:
@@ -492,6 +493,19 @@ def run_gradcam(
flush=True,
)
# Save raw CAM arrays
np.save(gradcam_dir / f"patient_{pid}_OD_cam.npy", cam_od)
np.save(gradcam_dir / f"patient_{pid}_OS_cam.npy", cam_os)
index_rows.append({
"patient_id": pid,
"true_label": label,
"true_name": true_name,
"pred_label": pred,
"pred_name": pred_name,
"confidence": conf,
"correct": correct,
})
# Accumulate for summary grid
od_overlay = overlay_gradcam(orig_od, cam_od, alpha) if orig_od else None
os_overlay = overlay_gradcam(orig_os, cam_os, alpha) if orig_os else None
@@ -500,6 +514,16 @@ def run_gradcam(
gcam.remove()
# ---- save index CSV ----
if index_rows:
import csv
idx_path = gradcam_dir / "gradcam_index.csv"
with idx_path.open("w", newline="") as f:
writer = csv.DictWriter(f, fieldnames=list(index_rows[0].keys()))
writer.writeheader()
writer.writerows(index_rows)
print(f" Index CSV → {idx_path}", flush=True)
# ---- summary grid: N_patients rows × 2 cols (OD overlay | OS overlay) ----
n = len(overlay_grid_items)
if n == 0:
@@ -611,84 +635,70 @@ def run_fusion_event_analysis(
pred_i = pi.argmax(axis=1)
pred_m = pm.argmax(axis=1)
corrections = (pred_f == y_true) & (pred_i != y_true) & (pred_m != y_true)
errors = (pred_f != y_true) & (pred_i == y_true) & (pred_m == y_true)
n_corr = corrections.sum()
n_err = errors.sum()
both_wrong = ((pred_i != y_true) & (pred_m != y_true)).sum()
both_correct = ((pred_i == y_true) & (pred_m == y_true)).sum()
# confidence of the predicted class for each head
conf_f = np.take_along_axis(pf, pred_f[:, None], axis=1).squeeze(1)
conf_i = np.take_along_axis(pi, pred_i[:, None], axis=1).squeeze(1)
conf_m = np.take_along_axis(pm, pred_m[:, None], axis=1).squeeze(1)
# how much did fusion shift confidence vs the average of the two towers?
conf_delta = conf_f - 0.5 * (conf_i + conf_m)
print(f" N={N} corrections={n_corr} errors={n_err} ratio={n_corr}/{n_err}", flush=True)
print(f" correction rate: {n_corr}/{both_wrong} = {n_corr/max(both_wrong,1):.2%} of both-wrong cases", flush=True)
print(f" error rate: {n_err}/{both_correct} = {n_err/max(both_correct,1):.2%} of both-correct cases", flush=True)
f_ok = pred_f == y_true
i_ok = pred_i == y_true
m_ok = pred_m == y_true
# ---- cache intermediate hm/hi vectors for all patients ----
model.eval()
hm1_list, hm2_list, hi1_list, hi2_list = [], [], [], []
with torch.no_grad():
for batch in loader:
x1 = batch.get("image_1"); m1 = batch.get("matrix_1")
x2 = batch.get("image_2"); m2 = batch.get("matrix_2")
if not (torch.is_tensor(x1) and torch.is_tensor(m1)):
continue
hi1 = model.bridge.ln_img(model.bridge.W_img(model.img_tower(x1.to(device))))
hi2 = model.bridge.ln_img(model.bridge.W_img(model.img_tower(x2.to(device))))
hm1 = model.bridge.ln_md(model.bridge.W_md(model.md_tower(m1.to(device))))
hm2 = model.bridge.ln_md(model.bridge.W_md(model.md_tower(m2.to(device))))
hi1_list.append(hi1.cpu()); hi2_list.append(hi2.cpu())
hm1_list.append(hm1.cpu()); hm2_list.append(hm2.cpu())
# 6 non-trivial bridge-effect event types
full_correction = f_ok & ~i_ok & ~m_ok # both towers wrong → fused right
img_assist = f_ok & ~i_ok & m_ok # img wrong, md right → fused right (md carried it)
md_assist = f_ok & i_ok & ~m_ok # md wrong, img right → fused right (img carried it)
full_error = ~f_ok & i_ok & m_ok # both towers right → fused wrong
img_drag = ~f_ok & ~i_ok & m_ok # img wrong, md right → fused wrong (img dragged it down)
md_drag = ~f_ok & i_ok & ~m_ok # md wrong, img right → fused wrong (md dragged it down)
concordant_ok = f_ok & i_ok & m_ok
concordant_bad = ~f_ok & ~i_ok & ~m_ok
hi1 = torch.cat(hi1_list) # [N, fusion_dim]
hi2 = torch.cat(hi2_list)
hm1 = torch.cat(hm1_list) # [N, fusion_dim]
hm2 = torch.cat(hm2_list)
event_labels = [
"full correction\n(both wrong→fused right)",
"img assist\n(img wrong, md right→right)",
"md assist\n(md wrong, img right→right)",
"full error\n(both right→fused wrong)",
"img drag\n(img wrong, md right→wrong)",
"md drag\n(md wrong, img right→wrong)",
]
event_masks = [full_correction, img_assist, md_assist, full_error, img_drag, md_drag]
event_colors = ["#2ca02c", "#98df8a", "#b5d46e", "#d62728", "#ff9896", "#ffbf9b"]
event_keys = ["full_correction", "img_assist", "md_assist",
"full_error", "img_drag", "md_drag"]
counts = [int(m.sum()) for m in event_masks]
hm1_mean = hm1.mean(dim=0, keepdim=True)
hm2_mean = hm2.mean(dim=0, keepdim=True)
print(f" N={N}", flush=True)
for label, count in zip(event_labels, counts):
print(f" {label.replace(chr(10), ' '):55s}: {count}", flush=True)
n_corr, n_err = counts[0], counts[3]
ratio_str = f"{n_corr}/{n_err}" if n_err > 0 else f"{n_corr}/0"
print(f" full correction/error ratio: {ratio_str}", flush=True)
print(f" conf_delta mean={conf_delta.mean():+.4f} median={np.median(conf_delta):+.4f}",
flush=True)
# ---- for each patient: compare logit[true_class] with real hm vs mean hm ----
gains = []
with torch.no_grad():
for idx in range(N):
true_cls = int(y_true[idx])
# patient-level average of OD/OS fused vectors (SE skipped: hard to replicate outside forward)
fused_real = (hi1[idx:idx+1] * hm1[idx:idx+1] + hi2[idx:idx+1] * hm2[idx:idx+1]) * 0.5
fused_mean = (hi1[idx:idx+1] * hm1_mean + hi2[idx:idx+1] * hm2_mean) * 0.5
logit_real = model.bridge.classifier_fused(fused_real.to(device))
logit_mean = model.bridge.classifier_fused(fused_mean.to(device))
gain = (logit_real[0, true_cls] - logit_mean[0, true_cls]).item()
gains.append(gain)
gains = np.array(gains)
if n_corr > 0:
corr_gains = gains[corrections]
helped = (corr_gains > 0).sum()
print(f"\n Fusion corrections — MD gate gain vs mean gate:", flush=True)
print(f" mean gain = {corr_gains.mean():+.4f} median = {np.median(corr_gains):+.4f}", flush=True)
print(f" real MD helped {helped}/{n_corr} correction patients ({helped/n_corr:.0%})", flush=True)
if n_err > 0:
err_gains = gains[errors]
print(f"\n Fusion errors — MD gate gain vs mean gate:", flush=True)
print(f" mean gain = {err_gains.mean():+.4f} median = {np.median(err_gains):+.4f}", flush=True)
# ---- save CSV ----
# ---- CSV ----
import csv
event_type = np.where(concordant_ok, "concordant_correct",
np.where(concordant_bad, "concordant_wrong", "other")).astype(object)
for mask, key in zip(event_masks, event_keys):
event_type[mask] = key
rows = []
for idx in range(N):
rows.append({
"patient_idx": idx,
"y_true": int(y_true[idx]),
"pred_fused": int(pred_f[idx]),
"pred_img": int(pred_i[idx]),
"pred_md": int(pred_m[idx]),
"conf_fused": float(pf[idx].max()),
"conf_img": float(pi[idx].max()),
"conf_md": float(pm[idx].max()),
"is_correction": bool(corrections[idx]),
"is_error": bool(errors[idx]),
"md_gate_gain": float(gains[idx]),
"patient_idx": idx,
"y_true": int(y_true[idx]),
"pred_fused": int(pred_f[idx]),
"pred_img": int(pred_i[idx]),
"pred_md": int(pred_m[idx]),
"conf_fused": float(conf_f[idx]),
"conf_img": float(conf_i[idx]),
"conf_md": float(conf_m[idx]),
"conf_delta": float(conf_delta[idx]),
"event_type": event_type[idx],
})
csv_path = out_dir / "fusion_events.csv"
with csv_path.open("w", newline="") as f:
@@ -697,25 +707,61 @@ def run_fusion_event_analysis(
writer.writerows(rows)
print(f" Saved → {csv_path}", flush=True)
# ---- chart ----
fig, axes = plt.subplots(1, 2, figsize=(11, 4))
# ---- plot ----
fig, axes = plt.subplots(1, 3, figsize=(15, 4))
categories = ["corrections\n(both wrong→fused right)", "errors\n(both right→fused wrong)"]
counts = [int(n_corr), int(n_err)]
axes[0].bar(categories, counts, color=["#e05c5c", "#5c9ee0"], width=0.5)
# Panel 1: stacked bar — positive events vs negative events
pos_counts = counts[:3]
neg_counts = counts[3:]
pos_colors = event_colors[:3]
neg_colors = event_colors[3:]
for bar_x, bar_counts, bar_colors in ((0, pos_counts, pos_colors),
(1, neg_counts, neg_colors)):
bot = 0
for c, col in zip(bar_counts, bar_colors):
axes[0].bar(bar_x, c, bottom=bot, color=col, width=0.5)
if c > 0:
axes[0].text(bar_x, bot + c / 2, str(c), ha="center", va="center",
fontsize=9, fontweight="bold")
bot += c
axes[0].set_xticks([0, 1])
axes[0].set_xticklabels(["Positive\nevents", "Negative\nevents"])
axes[0].set_ylabel("Count")
axes[0].set_title(f"Fusion Events (N={N})")
for i, v in enumerate(counts):
axes[0].text(i, v + 0.1, str(v), ha="center", fontsize=11)
patches = [mpatches.Patch(color=c, label=l.replace("\n", " "))
for c, l in zip(event_colors, event_labels)]
axes[0].legend(handles=patches, fontsize=6, loc="upper right")
if n_corr > 0:
axes[1].hist(gains[corrections], bins=10, alpha=0.7, color="#e05c5c", label=f"corrections (n={n_corr})")
if n_err > 0:
axes[1].hist(gains[errors], bins=10, alpha=0.7, color="#5c9ee0", label=f"errors (n={n_err})")
axes[1].axvline(0, color="black", linewidth=0.8)
axes[1].set_xlabel("MD gate gain vs mean gate\n(logit[true class]: real − mean)")
axes[1].set_title("Does real MD help the fused prediction?")
axes[1].legend(fontsize=9)
# Panel 2: conf_delta boxplot per event type (only non-empty)
box_data = [conf_delta[m] for m in event_masks if m.sum() > 0]
box_labels = [l.split("\n")[0] for m, l in zip(event_masks, event_labels) if m.sum() > 0]
box_cols = [c for m, c in zip(event_masks, event_colors) if m.sum() > 0]
if box_data:
bp = axes[1].boxplot(box_data, patch_artist=True, widths=0.5)
for patch, color in zip(bp["boxes"], box_cols):
patch.set_facecolor(color)
axes[1].set_xticks(range(1, len(box_labels) + 1))
axes[1].set_xticklabels(box_labels, rotation=35, ha="right", fontsize=7)
axes[1].axhline(0, color="black", linewidth=0.8, linestyle="--")
axes[1].set_ylabel("conf_delta\n(fused − avg(img, md))")
axes[1].set_title("Confidence delta by event type")
# Panel 3: img vs md confidence space, coloured by event type
for mask, color, label in zip(event_masks, event_colors, event_labels):
if mask.sum() > 0:
axes[2].scatter(conf_i[mask], conf_m[mask], c=color,
label=label.split("\n")[0], alpha=0.85, s=45, edgecolors="none")
if concordant_ok.sum() > 0:
axes[2].scatter(conf_i[concordant_ok], conf_m[concordant_ok],
c="lightgrey", alpha=0.4, s=20, edgecolors="none", label="concordant correct")
if concordant_bad.sum() > 0:
axes[2].scatter(conf_i[concordant_bad], conf_m[concordant_bad],
c="darkgrey", alpha=0.4, s=20, edgecolors="none", label="concordant wrong")
axes[2].plot([0, 1], [0, 1], "k--", linewidth=0.5, alpha=0.4)
axes[2].set_xlabel("conf_img")
axes[2].set_ylabel("conf_md")
axes[2].set_title("Tower confidence space\ncoloured by fusion event")
axes[2].legend(fontsize=6, loc="lower right")
fig.tight_layout()
fig.savefig(out_dir / "fusion_events.png", dpi=150)
scripts/output_analysis/explainability/explain_run.py
+1267
View File
File diff suppressed because it is too large Load Diff
scripts/output_analysis/explainability/plot_disc_attention_detail.py
+284
View File
@@ -0,0 +1,284 @@
#!/usr/bin/env python3
"""
Granular disc-attention visualisation.
Layout (3 rows × N_class cols):
Row 0 — disc-centred mean GradCAM patch for CORRECT predictions
Row 1 — disc-centred mean GradCAM patch for INCORRECT predictions
Row 2 — per-patient strip plot of disc_frac (blue=correct, red=incorrect)
Disc-centred patches: each patient's CAM is translated and scaled so the GT
disc centroid sits at the patch centre before averaging. A dashed white circle
marks the average GT disc size. This makes cross-patient averaging meaningful
regardless of where the disc sits in the original image.
Usage
-----
python scripts/output_analysis/explainability/plot_disc_attention_detail.py \
--agg-dir analysis_data/pipeline_nocrop/binary/single/gradcam_aggregate \
--manifest manifest.csv
"""
from __future__ import annotations
import argparse
from pathlib import Path
import numpy as np
import pandas as pd
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from matplotlib.patches import Circle
from PIL import Image, ImageDraw
DISC_SPAN = 5 # patch side = DISC_SPAN × disc diameter
OUTPUT_SIZE = 96 # pixel size of each thumbnail
# ---------------------------------------------------------------------------
# Disc mask helpers
# ---------------------------------------------------------------------------
def _load_disc_mask(contour_path: Path, orig_size: tuple, cam_h: int, cam_w: int):
try:
arr = np.loadtxt(str(contour_path), dtype=np.float32)
except Exception:
return None
if arr.ndim == 1:
arr = arr.reshape(-1, 2)
if arr.shape[0] < 3 or arr.shape[1] < 2:
return None
img = Image.new("L", orig_size, 0)
ImageDraw.Draw(img).polygon([tuple(pt) for pt in arr[:, :2]], fill=1)
return np.array(img.resize((cam_w, cam_h), Image.NEAREST), dtype=bool)
def build_disc_lookup(manifest_path: Path) -> dict:
mf = pd.read_csv(manifest_path)
lookup: dict = {}
for _, row in mf.iterrows():
sid = str(row["sample_id"])
if not sid.startswith("papila_RET"):
continue
suffix = sid[len("papila_RET"):]
eye = suffix[-2:]
pid = int(suffix[:-2])
disc_path = Path(str(row["annotation_disc"]))
img_path = Path(str(row["image_path"]))
if not disc_path.exists():
continue
try:
with Image.open(img_path) as im:
orig_size = im.size
except Exception:
continue
lookup[(pid, eye)] = (disc_path, orig_size)
return lookup
# ---------------------------------------------------------------------------
# Disc-centred patch extraction
# ---------------------------------------------------------------------------
def disc_centered_patch(
cam: np.ndarray,
disc_mask: np.ndarray,
span: int = DISC_SPAN,
out: int = OUTPUT_SIZE,
) -> tuple[np.ndarray | None, float | None]:
"""
Return (patch, disc_r_out):
patch — (out, out) float32 in [0, 1]
disc_r_out — disc radius in patch-pixel units (for drawing reference circle)
"""
if disc_mask is None or disc_mask.sum() == 0:
return None, None
ys, xs = np.where(disc_mask)
cy, cx = ys.mean(), xs.mean()
disc_r = float(np.sqrt(disc_mask.sum() / np.pi))
half = max(1, int(round(span * disc_r / 2)))
h, w = cam.shape
y0, y1 = int(round(cy)) - half, int(round(cy)) + half
x0, x1 = int(round(cx)) - half, int(round(cx)) + half
pt = max(0, -y0); pb = max(0, y1 - h)
pl = max(0, -x0); pr = max(0, x1 - w)
cam_pad = np.pad(cam, ((pt, pb), (pl, pr)), constant_values=0.0)
patch = cam_pad[y0 + pt : y1 + pt, x0 + pl : x1 + pl]
patch_img = Image.fromarray((np.clip(patch, 0, 1) * 255).astype(np.uint8))
patch_out = np.array(patch_img.resize((out, out), Image.BILINEAR)) / 255.0
disc_r_out = out * disc_r / (2 * half)
return patch_out.astype(np.float32), disc_r_out
# ---------------------------------------------------------------------------
# Data loading
# ---------------------------------------------------------------------------
def load_all_cam_records(mode_dir: Path) -> list[dict]:
records = []
fold_dirs = sorted(
[d for d in mode_dir.iterdir() if d.is_dir() and d.name.startswith("fold")],
key=lambda p: int(p.name.replace("fold", "")),
)
for fd in fold_dirs:
gcam_dir = fd / "explainability" / "gradcam"
idx_path = gcam_dir / "gradcam_index.csv"
if not idx_path.exists():
continue
idx = pd.read_csv(idx_path)
for _, row in idx.iterrows():
pid = int(row["patient_id"])
for eye in ("OD", "OS"):
npy = gcam_dir / f"patient_{pid}_{eye}_cam.npy"
if not npy.exists():
continue
records.append({
"patient_id": pid,
"eye": eye,
"true_name": row["true_name"],
"correct": bool(row["correct"]),
"cam": np.load(npy),
})
return records
def build_mean_patches(
records: list[dict],
disc_lookup: dict,
classes: list[str],
) -> dict[tuple, tuple]:
"""
Returns {(cls, split): (mean_patch, mean_disc_r_out, count)}
split = 'correct' | 'incorrect'
"""
buckets: dict[tuple, list] = {}
radii: dict[tuple, list] = {}
for r in records:
split = "correct" if r["correct"] else "incorrect"
key = (r["true_name"], split)
pid, eye = r["patient_id"], r["eye"]
if (pid, eye) not in disc_lookup:
continue
disc_path, orig_size = disc_lookup[(pid, eye)]
h, w = r["cam"].shape
disc_mask = _load_disc_mask(disc_path, orig_size, h, w)
patch, disc_r_out = disc_centered_patch(r["cam"], disc_mask)
if patch is None:
continue
buckets.setdefault(key, []).append(patch)
radii.setdefault(key, []).append(disc_r_out)
return {
key: (np.stack(ps).mean(0), float(np.mean(radii[key])), len(ps))
for key, ps in buckets.items()
}
# ---------------------------------------------------------------------------
# Plotting
# ---------------------------------------------------------------------------
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--agg-dir", required=True)
ap.add_argument("--manifest", default="manifest.csv")
ap.add_argument("--out", default=None)
args = ap.parse_args()
agg_dir = Path(args.agg_dir)
mode_dir = agg_dir.parent
out_path = Path(args.out) if args.out else agg_dir / "disc_attention_detail.png"
print("Loading disc lookup…")
disc_lookup = build_disc_lookup(Path(args.manifest))
print(f" {len(disc_lookup)} entries")
print("Loading CAM records…")
records = load_all_cam_records(mode_dir)
print(f" {len(records)} eye records")
stats = pd.read_csv(agg_dir / "attention_stats.csv")
classes = sorted({r["true_name"] for r in records})
n_cls = len(classes)
print(f"Classes: {classes}")
print("Building disc-centred mean patches…")
mean_patches = build_mean_patches(records, disc_lookup, classes)
# -----------------------------------------------------------------------
# Figure
# -----------------------------------------------------------------------
corr_colors = {"correct": "steelblue", "incorrect": "tomato"}
splits = ["correct", "incorrect"]
row_labels = ["Correct", "Incorrect", "Disc fraction\n(strip plot)"]
fig, axes = plt.subplots(3, n_cls, figsize=(4.2 * n_cls, 13))
if n_cls == 1:
axes = axes[:, np.newaxis]
# ---- rows 0 & 1: disc-centred heatmaps ----
for ri, split in enumerate(splits):
for ci, cls in enumerate(classes):
ax = axes[ri, ci]
key = (cls, split)
if key in mean_patches:
mean_patch, disc_r_out, count = mean_patches[key]
ax.imshow(mean_patch, cmap="jet", vmin=0, vmax=1, origin="upper",
extent=[0, OUTPUT_SIZE, OUTPUT_SIZE, 0])
cx = cy = OUTPUT_SIZE / 2
ax.add_patch(Circle((cx, cy), disc_r_out,
fill=False, edgecolor="white",
linewidth=2, linestyle="--"))
ax.set_title(f"{cls} | {split}\n(N={count})", fontsize=9)
else:
ax.text(0.5, 0.5, "no data", ha="center", va="center",
transform=ax.transAxes, fontsize=9, color="grey")
ax.set_title(f"{cls} | {split}", fontsize=9)
ax.axis("off")
axes[ri, 0].set_ylabel(row_labels[ri], fontsize=10, labelpad=6)
# ---- row 2: strip plots ----
rng = np.random.default_rng(42)
for ci, cls in enumerate(classes):
ax = axes[2, ci]
sub = stats[stats["true_name"] == cls].dropna(subset=["disc_frac"])
for xi, split in enumerate(splits):
correct_val = (split == "correct")
pts = sub[sub["correct"] == correct_val]["disc_frac"].values
if len(pts) == 0:
continue
color = corr_colors[split]
jitter = rng.uniform(-0.18, 0.18, size=len(pts))
ax.scatter(xi + jitter, pts, color=color, alpha=0.7, s=28, edgecolors="none")
ax.hlines(pts.mean(), xi - 0.28, xi + 0.28,
colors=color, linewidth=2.5, zorder=5)
ax.set_xticks([0, 1])
ax.set_xticklabels(["Correct", "Incorrect"], fontsize=9)
ax.set_xlim(-0.55, 1.55)
ax.set_ylim(0, 1)
ax.set_title(cls, fontsize=10)
ax.grid(axis="y", linestyle="--", alpha=0.3)
if ci == 0:
ax.set_ylabel("Disc fraction\n(GT disc attention)", fontsize=9)
fig.suptitle(
"Disc-centred GradCAM attention | dashed circle = GT disc boundary",
fontsize=12,
)
fig.tight_layout()
fig.savefig(out_path, dpi=150, bbox_inches="tight")
plt.close(fig)
print(f"Saved → {out_path}")
if __name__ == "__main__":
main()
scripts/output_analysis/explainability/plot_gradcam_aggregate.py
+286
View File
@@ -0,0 +1,286 @@
#!/usr/bin/env python3
"""
Visualise aggregated GradCAM heatmaps produced by aggregate_gradcam.py.
Produces two figures:
Figure 1 — Mean heatmaps grid
Rows: classes (e.g. Normal, Glaucoma)
Cols: OD_all | OS_all | OD_correct | OD_incorrect | OS_correct | OS_incorrect
Figure 2 — Attention stats
Panel A: disc_frac distribution per class (violin/box), OD and OS side by side
Panel B: entropy distribution per class
Panel C: disc_frac correct vs incorrect per class (scatter means + error bars)
Figure 3 — Disc attention vs correct confidence
Scatter of disc_frac vs correct_conf (confidence if correct, 1-confidence if wrong)
One panel per class, OD and OS overlaid, Pearson r annotated
Usage
-----
python scripts/output_analysis/explainability/plot_gradcam_aggregate.py \
--agg-dir analysis_data/pipeline_nocrop/binary/single/gradcam_aggregate
"""
from __future__ import annotations
import argparse
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import numpy as np
import pandas as pd
def load_agg(agg_dir: Path):
npz = np.load(agg_dir / "mean_heatmaps.npz")
stats = pd.read_csv(agg_dir / "attention_stats.csv")
return npz, stats
def _classes_from_npz(npz) -> list[str]:
classes = []
for key in npz.files:
parts = key.split("_")
# key format: {EYE}_{ClassName}_{split}_{stat}
# ClassName may be multi-word (e.g. "Glaucoma", "Normal", "Suspect")
if parts[-1] == "mean" and parts[-2] == "all" and parts[0] == "OD":
classes.append(parts[1])
return sorted(set(classes))
def plot_mean_heatmaps(npz, classes: list[str], out_path: Path):
eyes = ["OD", "OS"]
splits = ["all", "correct", "incorrect"]
cols = [(e, s) for e in eyes for s in splits] # 6 columns
n_rows = len(classes)
n_cols = len(cols)
fig, axes = plt.subplots(n_rows, n_cols, figsize=(n_cols * 2.8, n_rows * 2.8))
if n_rows == 1:
axes = axes[np.newaxis, :]
for r, cls in enumerate(classes):
for c, (eye, split) in enumerate(cols):
ax = axes[r, c]
key = f"{eye}_{cls}_{split}_mean"
if key not in npz:
ax.axis("off")
ax.set_title(f"{eye} {split}\n(no data)", fontsize=7)
continue
cam = npz[key]
count = int(npz.get(f"{eye}_{cls}_{split}_count", np.array(0)))
ax.imshow(cam, cmap="jet", vmin=0, vmax=1)
ax.axis("off")
title = f"{cls} | {eye} {split}\n(N={count})"
ax.set_title(title, fontsize=7)
fig.suptitle("Mean GradCAM heatmaps by class / eye / outcome", fontsize=12)
fig.tight_layout()
fig.savefig(out_path, dpi=150, bbox_inches="tight")
plt.close(fig)
print(f"Saved → {out_path}")
def plot_attention_stats(stats: pd.DataFrame, classes: list[str], out_path: Path):
eyes = ["OD", "OS"]
cmap = plt.get_cmap("tab10")
class_colors = {cls: cmap(i) for i, cls in enumerate(classes)}
fig, axes = plt.subplots(1, 3, figsize=(16, 5))
# ---- Panel A: disc_frac per class × eye ----
ax = axes[0]
positions = []
labels = []
data_viol = []
tick_pos = []
pos = 0
for cls in classes:
for eye in eyes:
sub = stats[(stats["true_name"] == cls) & (stats["eye"] == eye)]["disc_frac"].dropna()
data_viol.append(sub.values)
positions.append(pos)
labels.append(f"{cls[:3]}\n{eye}")
tick_pos.append(pos)
pos += 1
pos += 0.5 # gap between classes
vp = ax.violinplot(data_viol, positions=positions, showmedians=True, widths=0.7)
for i, (pc, cls) in enumerate(zip(vp["bodies"], [c for c in classes for _ in eyes])):
pc.set_facecolor(class_colors[cls])
pc.set_alpha(0.65)
ax.set_xticks(tick_pos)
ax.set_xticklabels(labels, fontsize=8)
ax.set_ylabel("Disc fraction (attention mass within GT disc mask)")
ax.set_title("Disc attention by class")
ax.axhline(0.5, color="black", linewidth=0.8, linestyle="--", alpha=0.4)
ax.grid(axis="y", linestyle="--", alpha=0.3)
# ---- Panel B: entropy per class × eye (same layout) ----
ax = axes[1]
data_ent = []
for cls in classes:
for eye in eyes:
sub = stats[(stats["true_name"] == cls) & (stats["eye"] == eye)]["entropy"].dropna()
data_ent.append(sub.values)
vp2 = ax.violinplot(data_ent, positions=positions, showmedians=True, widths=0.7)
for pc, cls in zip(vp2["bodies"], [c for c in classes for _ in eyes]):
pc.set_facecolor(class_colors[cls])
pc.set_alpha(0.65)
ax.set_xticks(tick_pos)
ax.set_xticklabels(labels, fontsize=8)
ax.set_ylabel("Attention entropy (higher = more diffuse)")
ax.set_title("Attention entropy by class")
ax.grid(axis="y", linestyle="--", alpha=0.3)
# ---- Panel C: disc_frac correct vs incorrect, mean ± std ----
ax = axes[2]
x_ticks = []
x_labels = []
pos = 0
for cls in classes:
for eye in eyes:
for split, marker, ls in [("correct", "o", "-"), ("incorrect", "X", "--")]:
sub = stats[
(stats["true_name"] == cls) &
(stats["eye"] == eye) &
(stats["correct"] == (split == "correct"))
]["disc_frac"].dropna()
if len(sub) == 0:
continue
ax.errorbar(
pos, sub.mean(), yerr=sub.std(),
fmt=marker, color=class_colors[cls], linestyle=ls,
capsize=4, markersize=7, alpha=0.85,
label=f"{cls[:3]} {eye} {split}" if pos < 4 else "_",
)
pos += 1
x_ticks.append(pos - 1.5)
x_labels.append(f"{cls[:3]}\n{eye}")
pos += 0.5
ax.axhline(0.5, color="black", linewidth=0.8, linestyle="--", alpha=0.4)
ax.set_ylabel("Disc fraction")
ax.set_title("Disc fraction: correct vs incorrect\n(circle=correct, X=incorrect)")
ax.grid(axis="y", linestyle="--", alpha=0.3)
# legend: one patch per class
patches = [mpatches.Patch(color=class_colors[c], label=c) for c in classes]
patches += [
plt.Line2D([0], [0], marker="o", color="grey", label="correct", linestyle="none"),
plt.Line2D([0], [0], marker="X", color="grey", label="incorrect", linestyle="none"),
]
ax.legend(handles=patches, fontsize=7, loc="lower right")
fig.suptitle("GradCAM attention statistics", fontsize=12)
fig.tight_layout()
fig.savefig(out_path, dpi=150, bbox_inches="tight")
plt.close(fig)
print(f"Saved → {out_path}")
def plot_disc_attention_correlation(stats: pd.DataFrame, classes: list[str], out_path: Path):
"""
Scatter disc_frac vs correct_conf per class.
correct_conf = confidence if correct
= 1 - confidence if incorrect
This asks: does focusing attention on the disc region correlate with
the model being more confident about the right answer?
"""
import scipy.stats as scipy_stats
stats = stats.copy()
stats["correct_conf"] = np.where(
stats["correct"],
stats["confidence"],
1.0 - stats["confidence"],
)
corr_colors = {True: "steelblue", False: "tomato"}
corr_labels = {True: "Correct", False: "Incorrect"}
is_binary = len(classes) == 2
n_cls = len(classes)
fig, axes = plt.subplots(1, n_cls, figsize=(5 * n_cls, 5), sharey=True)
if n_cls == 1:
axes = [axes]
for ax, cls in zip(axes, classes):
sub = stats[stats["true_name"] == cls]
x_all, y_all = [], []
for correct_val, color in corr_colors.items():
csub = sub[sub["correct"] == correct_val]
x = csub["disc_frac"].values
y = csub["correct_conf"].values
ax.scatter(x, y, marker="o", color=color,
alpha=0.75, s=30,
label=corr_labels[correct_val],
edgecolors="none")
x_all.extend(x.tolist())
y_all.extend(y.tolist())
# pooled regression line
x_arr = np.array(x_all)
y_arr = np.array(y_all)
if len(x_arr) >= 3:
r, p = scipy_stats.pearsonr(x_arr, y_arr)
m, b = np.polyfit(x_arr, y_arr, 1)
xs = np.linspace(0, 1, 100)
ax.plot(xs, m * xs + b, color="black", linewidth=1.5, linestyle="--", alpha=0.7)
p_str = f"p={p:.3f}" if p >= 0.001 else "p<0.001"
ax.annotate(f"r={r:+.3f}\n{p_str}", xy=(0.05, 0.93), xycoords="axes fraction",
fontsize=9, va="top",
bbox=dict(boxstyle="round,pad=0.3", facecolor="white", alpha=0.7))
if is_binary:
ax.axhline(0.5, color="red", linewidth=1.0, linestyle=":",
alpha=0.7, label="Decision boundary (0.50)")
ax.set_xlim(0, 1)
ax.set_xlabel("Disc fraction\n(attention mass within GT disc mask)", fontsize=9)
ax.set_title(cls, fontsize=11)
ax.set_ylim(-0.02, 1.05)
ax.grid(linestyle="--", alpha=0.3)
ax.legend(fontsize=8, loc="lower right")
axes[0].set_ylabel("Correct-class confidence\n(conf if correct, 1−conf if wrong)", fontsize=9)
fig.suptitle("Disc attention vs correct-class confidence", fontsize=12)
fig.tight_layout()
fig.savefig(out_path, dpi=150, bbox_inches="tight")
plt.close(fig)
print(f"Saved → {out_path}")
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--agg-dir", required=True,
help="Directory produced by aggregate_gradcam.py")
ap.add_argument("--out-heatmaps", default=None)
ap.add_argument("--out-stats", default=None)
ap.add_argument("--out-corr", default=None)
args = ap.parse_args()
agg_dir = Path(args.agg_dir)
out_hm = Path(args.out_heatmaps) if args.out_heatmaps else agg_dir / "mean_heatmaps_plot.png"
out_st = Path(args.out_stats) if args.out_stats else agg_dir / "attention_stats_plot.png"
out_corr = Path(args.out_corr) if args.out_corr else agg_dir / "disc_attention_correlation.png"
npz, stats = load_agg(agg_dir)
classes = _classes_from_npz(npz)
print(f"Classes found: {classes}")
print(f"Total eye records in stats: {len(stats)}")
plot_mean_heatmaps(npz, classes, out_hm)
plot_attention_stats(stats, classes, out_st)
plot_disc_attention_correlation(stats, classes, out_corr)
if __name__ == "__main__":
main()
scripts/output_analysis/reeval_holdout_threshold.py
+355
View File
@@ -0,0 +1,355 @@
#!/usr/bin/env python3
"""
Re-evaluate holdout accuracy using two threshold strategies:
1. acc — current behaviour: maximise raw accuracy on (imbalanced) val set
2. youden — Youden's J = sensitivity + specificity − 1 on val set
For each fold the val probs (already saved) supply the threshold, then the
model is re-run on the holdout set to get the actual holdout accuracy under
each strategy.
Usage
-----
python scripts/output_analysis/reeval_holdout_threshold.py \
--run-dir analysis_data/pipeline_10x5 \
--eval-mode binary
# or a single nocrop run:
python scripts/output_analysis/reeval_holdout_threshold.py \
--run-dir analysis_data/pipeline_nocrop \
--eval-mode binary \
--fold-seed 42
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from types import SimpleNamespace
import numpy as np
import pandas as pd
import torch
from sklearn.metrics import balanced_accuracy_score, roc_auc_score, roc_curve
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.v2.metrics import tune_multiclass_bias
from classes.v2.papila_builders import build_papila_data
from classes.v2.loader_factory import filter_bilateral_samples, make_loader
from classes.v2.models import SingleEyeHT, collect_probs_single_components
from classes.v2.profiles import build_papila_profile
from classes.v2.split_manager import PatientFirstSplitManager
from classes.v2.transforms import build_eval_transform
from classes.v2.utils import choose_device
# ---------------------------------------------------------------------------
# Threshold helpers
# ---------------------------------------------------------------------------
def _acc_threshold(y: np.ndarray, p1: np.ndarray) -> float:
grid = np.linspace(0.0, 1.0, 1001)
best_t, best_acc = 0.5, -1.0
for t in grid:
acc = float(((p1 >= t).astype(int) == y).mean())
if acc > best_acc or (acc == best_acc and abs(t - 0.5) < abs(best_t - 0.5)):
best_acc, best_t = acc, float(t)
return best_t
def _youden_threshold(y: np.ndarray, p1: np.ndarray) -> float:
if len(np.unique(y)) < 2:
return 0.5
fpr, tpr, thresholds = roc_curve(y, p1)
j = tpr + (1.0 - fpr) - 1.0
return float(thresholds[np.argmax(j)])
def _apply_threshold(probs: np.ndarray, threshold: float, num_classes: int) -> np.ndarray:
if num_classes == 2:
return (probs[:, 1] >= threshold).astype(int)
# multiclass: not applicable for a single scalar threshold
return probs.argmax(axis=1)
# ---------------------------------------------------------------------------
# Per-fold evaluation
# ---------------------------------------------------------------------------
def eval_fold(
fold_dir: Path,
fold_idx: int,
fold_seed: int,
eval_mode: str,
args,
device: torch.device,
) -> dict | None:
checkpoint = fold_dir / "best_single.pt"
if not checkpoint.exists():
print(f" [skip] {fold_dir}: no best_single.pt")
return None
val_y_path = fold_dir / "y_true.npy"
val_p_path = fold_dir / "probs_fused.npy"
if not val_y_path.exists() or not val_p_path.exists():
print(f" [skip] {fold_dir}: no val probs")
return None
val_y = np.load(val_y_path)
val_p = np.load(val_p_path)
num_classes = val_p.shape[1]
# ---- decision boundaries from val ----
if num_classes == 2:
t_acc = _acc_threshold(val_y, val_p[:, 1])
t_youden = _youden_threshold(val_y, val_p[:, 1])
else:
# multiclass: compare raw-acc-optimised bias vs balanced-acc-optimised bias
# raw-acc bias: temporarily swap objective back to raw accuracy
from sklearn.metrics import accuracy_score
import copy
def _tune_bias_raw(y, p):
c = p.shape[1]
bias = np.zeros(c)
grid = np.linspace(-1.0, 1.0, 41)
for _ in range(2):
for k in range(c):
best_v, best_acc = bias[k], -1.0
old = bias[k]
for v in grid:
bias[k] = float(v)
logits = np.log(np.clip(p, 1e-8, 1.0)) + bias.reshape(1, -1)
acc = float((logits.argmax(1) == y).mean())
if acc > best_acc or (acc == best_acc and abs(v) < abs(best_v)):
best_acc, best_v = acc, float(v)
bias[k] = best_v
return bias
bias_raw = _tune_bias_raw(val_y, val_p)
bias_bal = tune_multiclass_bias(val_y, val_p) # balanced acc objective
# ---- reconstruct holdout split ----
data = build_papila_data(
image_dir=args.image_dir,
clinical_dir=args.clinical_dir,
label_col=args.label_col,
cat_cols=args.cat_cols,
n_splits=args.n_splits,
random_seed=fold_seed,
iop_corr_method=getattr(args, "iop_corr_method", "ratio"),
)
df_mode = data.df.copy()
if eval_mode == "binary":
df_mode = df_mode[df_mode[args.label_col].isin([0, 1])].reset_index(drop=True)
splitter = PatientFirstSplitManager(
patient_col="Patient ID", label_col=args.label_col
)
split_args = SimpleNamespace(
eval_mode=eval_mode,
holdout_per_class=args.holdout_per_class,
holdout_seed=args.holdout_seed,
n_splits=args.n_splits,
fold_seed=fold_seed,
)
plans = splitter.build_plans(
clinical=SimpleNamespace(df=df_mode, label_col=args.label_col),
args=split_args,
profile=None,
)
split = plans[fold_idx]
if split.holdout is None or split.holdout.empty:
print(f" [skip] {fold_dir}: no holdout")
return None
# ---- build holdout loader ----
profile_patient = build_papila_profile(
patient_col="Patient ID", label_col=args.label_col, sample_mode="patient"
)
holdout_samples = filter_bilateral_samples(
profile_patient.build_samples(df=split.holdout, clinical=data)
)
if not holdout_samples:
print(f" [skip] {fold_dir}: no bilateral holdout samples")
return None
eval_transform = build_eval_transform(args.backbone)
holdout_loader = make_loader(
holdout_samples,
profile_patient.slot_descriptors(),
image_transform=eval_transform,
image_preprocessor=None,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
)
# ---- load model ----
model = SingleEyeHT(
backbone=args.backbone,
freeze_ratio=0.0,
augment=False,
clinical_data=data,
num_classes=num_classes,
md_hidden_dim=getattr(args, "md_hidden_dim", 64),
fusion_dim=getattr(args, "fusion_dim", 128),
bridge_mode=getattr(args, "bridge_mode", "fused"),
).to(device)
model.load_state_dict(
torch.load(checkpoint, map_location=device, weights_only=False)
)
model.eval()
# ---- run inference ----
hld_y, hld_p, _, _ = collect_probs_single_components(
model, holdout_loader, device, aggregate_patient=True
)
if len(hld_y) == 0:
return None
hld_auc = float(roc_auc_score(
hld_y, hld_p[:, 1] if num_classes == 2 else hld_p,
multi_class="ovr" if num_classes > 2 else "raise",
))
if num_classes == 2:
acc_old = float(((hld_p[:, 1] >= t_acc).astype(int) == hld_y).mean())
acc_new = float(((hld_p[:, 1] >= t_youden).astype(int) == hld_y).mean())
bacc_old = balanced_accuracy_score(hld_y, (hld_p[:, 1] >= t_acc).astype(int))
bacc_new = balanced_accuracy_score(hld_y, (hld_p[:, 1] >= t_youden).astype(int))
row_extra = {"t_old": t_acc, "t_new": t_youden}
else:
logits_raw = np.log(np.clip(hld_p, 1e-8, 1.0)) + bias_raw.reshape(1, -1)
logits_bal = np.log(np.clip(hld_p, 1e-8, 1.0)) + bias_bal.reshape(1, -1)
preds_raw = logits_raw.argmax(1)
preds_bal = logits_bal.argmax(1)
acc_old = float((preds_raw == hld_y).mean())
acc_new = float((preds_bal == hld_y).mean())
bacc_old = balanced_accuracy_score(hld_y, preds_raw)
bacc_new = balanced_accuracy_score(hld_y, preds_bal)
row_extra = {"bias_raw": bias_raw.tolist(), "bias_bal": bias_bal.tolist()}
return {
"fold_dir": str(fold_dir),
"fold": fold_idx,
"fold_seed": fold_seed,
"hld_auc": hld_auc,
"hld_acc_old": acc_old,
"hld_acc_new": acc_new,
"hld_bacc_old": bacc_old,
"hld_bacc_new": bacc_new,
"n_holdout": len(hld_y),
**row_extra,
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--run-dir", required=True,
help="e.g. analysis_data/pipeline_10x5 or analysis_data/pipeline_nocrop")
ap.add_argument("--eval-mode", default="binary", choices=["binary", "multiclass"])
ap.add_argument("--fold-seed", type=int, default=None,
help="Override fold seed (for single-rep runs). "
"For 10x5, seeds are inferred from rep dir name.")
ap.add_argument("--backbone", default="refugelike")
ap.add_argument("--n-splits", type=int, default=5)
ap.add_argument("--holdout-per-class", type=int, default=5)
ap.add_argument("--holdout-seed", type=int, default=123)
ap.add_argument("--label-col", default="Diagnosis")
ap.add_argument("--cat-cols", nargs="*", default=["Gender"])
ap.add_argument("--image-dir", default="Papila/FundusImages")
ap.add_argument("--clinical-dir",default="Papila/ClinicalData")
ap.add_argument("--iop-corr-method", default="ratio")
ap.add_argument("--batch-size", type=int, default=8)
ap.add_argument("--num-workers", type=int, default=4)
ap.add_argument("--md-hidden-dim", type=int, default=128)
ap.add_argument("--fusion-dim", type=int, default=256)
ap.add_argument("--bridge-mode", default="fused")
ap.add_argument("--device", default="auto")
ap.add_argument("--out", default=None)
args = ap.parse_args()
device = choose_device(args.device)
run_dir = Path(args.run_dir)
out_path = Path(args.out) if args.out else \
run_dir / f"reeval_threshold_{args.eval_mode}.csv"
# Discover fold dirs — supports both flat (fold0..fold4) and
# rep-based (rep00/binary/ensemble/fold0) layouts
_BASE_SEED = 100
_SEED_STRIDE = 100
fold_jobs: list[tuple[Path, int, int]] = [] # (fold_dir, fold_idx, fold_seed)
rep_dirs = sorted(run_dir.glob("rep[0-9]*"))
if rep_dirs:
for rep_dir in rep_dirs:
rep_n = int(rep_dir.name.replace("rep", ""))
fold_seed = _BASE_SEED + rep_n * _SEED_STRIDE
mode_dir = rep_dir / args.eval_mode / "ensemble"
if not mode_dir.exists():
continue
for fd in sorted(mode_dir.glob("fold[0-9]*"), key=lambda p: int(p.name[4:])):
fold_jobs.append((fd, int(fd.name[4:]), fold_seed))
else:
# flat layout
mode_dir = run_dir / args.eval_mode / "ensemble"
fold_seed = args.fold_seed if args.fold_seed is not None else 42
for fd in sorted(mode_dir.glob("fold[0-9]*"), key=lambda p: int(p.name[4:])):
fold_jobs.append((fd, int(fd.name[4:]), fold_seed))
if not fold_jobs:
sys.exit(f"No fold directories found under {run_dir}")
print(f"Found {len(fold_jobs)} folds to re-evaluate")
rows = []
for i, (fold_dir, fold_idx, fold_seed) in enumerate(fold_jobs):
print(f"\n[{i+1}/{len(fold_jobs)}] {fold_dir} fold_seed={fold_seed}")
row = eval_fold(fold_dir, fold_idx, fold_seed, args.eval_mode, args, device)
if row:
rows.append(row)
print(f" hld_acc(old)={row['hld_acc_old']:.3f} "
f"hld_acc(new)={row['hld_acc_new']:.3f} "
f"hld_bacc(old)={row['hld_bacc_old']:.3f} "
f"hld_bacc(new)={row['hld_bacc_new']:.3f} "
f"hld_auc={row['hld_auc']:.3f}")
if not rows:
print("No results.")
return
df = pd.DataFrame(rows)
df.to_csv(out_path, index=False)
print(f"\nSaved → {out_path}")
is_binary = args.eval_mode == "binary"
old_label = "acc-threshold" if is_binary else "raw-acc bias"
new_label = "Youden-J" if is_binary else "balanced-acc bias"
print(f"\n{'='*60}")
print(f"Summary ({args.eval_mode}, n={len(df)} folds)")
print(f"{'='*60}")
print(f" Holdout AUC: {df.hld_auc.mean():.4f} ± {df.hld_auc.std():.4f}")
print(f" Holdout acc ({old_label:<18}): {df.hld_acc_old.mean():.4f} ± {df.hld_acc_old.std():.4f}")
print(f" Holdout acc ({new_label:<18}): {df.hld_acc_new.mean():.4f} ± {df.hld_acc_new.std():.4f}")
print(f" Holdout bacc ({old_label:<18}): {df.hld_bacc_old.mean():.4f} ± {df.hld_bacc_old.std():.4f}")
print(f" Holdout bacc ({new_label:<18}): {df.hld_bacc_new.mean():.4f} ± {df.hld_bacc_new.std():.4f}")
print(f" Delta acc (new − old): {df.hld_acc_new.mean() - df.hld_acc_old.mean():+.4f}")
print(f" Delta bacc (new − old): {df.hld_bacc_new.mean() - df.hld_bacc_old.mean():+.4f}")
if __name__ == "__main__":
main()
scripts/output_analysis/visualizations/build_10x5_aggregate_panel.py
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
"""
Assemble the 10×5 aggregate comparison panel.
Layout (3 rows × 2 cols):
col 0 = binary, col 1 = multiclass
row 0: mean ROC curve
row 1: rep stability
row 2: holdout stability
Usage
-----
python scripts/output_analysis/visualizations/build_10x5_aggregate_panel.py \
--run-dir analysis_data/pipeline_10x5 \
--out analysis_data/pipeline_10x5/aggregate/aggregate_panel.png
"""
from __future__ import annotations
import argparse
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
# (row, col, rel_path, label)
CELLS = [
(0, 0, "aggregate/binary_roc_mean.png", "Binary — Mean ROC"),
(0, 1, "aggregate/multiclass_roc_mean.png", "Multiclass — Mean ROC"),
(1, 0, "aggregate/binary_rep_stability.png", "Binary — Rep stability"),
(1, 1, "aggregate/multiclass_rep_stability.png", "Multiclass — Rep stability"),
(2, 0, "aggregate/binary_holdout_stability.png", "Binary — Holdout stability"),
(2, 1, "aggregate/multiclass_holdout_stability.png", "Multiclass — Holdout stability"),
]
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--run-dir", default="analysis_data/pipeline_10x5")
ap.add_argument("--out", default=None)
args = ap.parse_args()
run_dir = Path(args.run_dir)
out = Path(args.out) if args.out else run_dir / "aggregate" / "aggregate_panel.png"
fig = plt.figure(figsize=(16, 18))
gs = fig.add_gridspec(3, 2, hspace=0.06, wspace=0.04)
for row, col, rel, label in CELLS:
ax = fig.add_subplot(gs[row, col])
img = mpimg.imread(str(run_dir / rel))
ax.imshow(img)
ax.axis("off")
ax.set_title(label, fontsize=11, pad=5)
out.parent.mkdir(parents=True, exist_ok=True)
fig.savefig(out, dpi=150, bbox_inches="tight")
plt.close(fig)
print(f"Saved → {out}")
if __name__ == "__main__":
main()
scripts/output_analysis/visualizations/build_10x5_training_panel.py
+86
View File
@@ -0,0 +1,86 @@
#!/usr/bin/env python3
"""
Assemble the 10×5 training dynamics panel.
Layout: 3-row × 4-col gridspec with spanning
Top 2×2 (each cell spans 2 cols):
row 0, cols 0-1: binary early stopping sweep
row 0, cols 2-3: multiclass early stopping sweep
row 1, cols 0-1: binary cost of stopping early
row 1, cols 2-3: multiclass cost of stopping early
Bottom row of 4 (one col each):
row 2, col 0: binary holdout AUC by epoch
row 2, col 1: binary val−holdout gap (val-adjusted)
row 2, col 2: multiclass holdout AUC by epoch
row 2, col 3: multiclass val−holdout gap (val-adjusted)
Usage
-----
python scripts/output_analysis/visualizations/build_10x5_training_panel.py \
--run-dir analysis_data/pipeline_10x5 \
--out analysis_data/pipeline_10x5/aggregate/training_dynamics_panel.png
"""
from __future__ import annotations
import argparse
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
# (row, col_start, col_end, rel_path, label)
# col_end is exclusive slice — use None for single cell
CELLS = [
# ---- top 2×2: early stopping (each spans 2 cols) ----
(0, 0, 2, "binary/ensemble/plots/early_stopping_sweep_fused.png",
"Binary — Early stopping sweep"),
(0, 2, 4, "multiclass/ensemble/plots/early_stopping_sweep_fused.png",
"Multiclass — Early stopping sweep"),
(1, 0, 2, "binary/ensemble/plots/early_stopping_sweep_fused_inverted.png",
"Binary — Cost of stopping early"),
(1, 2, 4, "multiclass/ensemble/plots/early_stopping_sweep_fused_inverted_tol0.002.png",
"Multiclass — Cost of stopping early (CI tol=0.002)"),
# ---- bottom row of 4: holdout epoch curves (single col each) ----
(2, 0, 1, "binary/ensemble/plots/holdout_epoch_curves_fused.png",
"Binary — Holdout AUC by epoch"),
(2, 1, 2, "binary/ensemble/plots/holdout_epoch_curves_fused_delta_adj.png",
"Binary — Val−Holdout gap (val-adjusted)"),
(2, 2, 3, "multiclass/ensemble/plots/holdout_epoch_curves_fused.png",
"Multiclass — Holdout AUC by epoch"),
(2, 3, 4, "multiclass/ensemble/plots/holdout_epoch_curves_fused_delta_adj.png",
"Multiclass — Val−Holdout gap (val-adjusted)"),
]
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--run-dir", default="analysis_data/pipeline_10x5")
ap.add_argument("--out", default=None)
args = ap.parse_args()
run_dir = Path(args.run_dir)
out = Path(args.out) if args.out else run_dir / "aggregate" / "training_dynamics_panel.png"
fig = plt.figure(figsize=(22, 16))
gs = fig.add_gridspec(3, 4, height_ratios=[1, 1, 0.75], hspace=0.08, wspace=0.04)
for row, col_start, col_end, rel, label in CELLS:
ax = fig.add_subplot(gs[row, col_start:col_end])
img = mpimg.imread(str(run_dir / rel))
ax.imshow(img)
ax.axis("off")
ax.set_title(label, fontsize=11, pad=5)
out.parent.mkdir(parents=True, exist_ok=True)
fig.savefig(out, dpi=150, bbox_inches="tight")
plt.close(fig)
print(f"Saved → {out}")
if __name__ == "__main__":
main()
scripts/output_analysis/visualizations/build_fusion_head_panel.py
+94
View File
@@ -0,0 +1,94 @@
#!/usr/bin/env python3
"""
Assemble a 2x2 fusion-head comparison panel from pipeline_nocrop.
Layout:
[binary ensemble ROC] [binary fusion-head explainability summary]
[multiclass ensemble ROC][multiclass fusion-head explainability summary]
Usage
-----
python scripts/output_analysis/visualizations/build_fusion_head_panel.py \
--run-dir analysis_data/pipeline_nocrop \
--out analysis_data/pipeline_nocrop/fusion_head_comparison_panel.png
"""
from __future__ import annotations
import argparse
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import numpy as np
ROC_CELLS = [
# (row, col, rel_path, label)
(0, 0, "binary/ensemble/plots/roc_probs_fused_mean_ovr.png",
"Binary — Ensemble"),
(0, 1, "binary/ensemble/plots/roc_probs_fused_head_mean_ovr.png",
"Binary — Fusion Head"),
(1, 0, "multiclass/ensemble/plots/roc_probs_fused_mean_ovr.png",
"Multiclass — Ensemble"),
(1, 1, "multiclass/ensemble/plots/roc_probs_fused_head_mean_ovr.png",
"Multiclass — Fusion Head"),
]
EXPL_ROWS = [
# (row_in_grid, rel_path, label)
(2, "binary/ensemble/explainability_fusion_summary_val_fused_head.png",
"Binary — Fusion Head events (val)"),
(3, "multiclass/ensemble/explainability_fusion_summary_val_fused_head.png",
"Multiclass — Fusion Head events (val)"),
]
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--run-dir", default="analysis_data/pipeline_nocrop")
ap.add_argument("--out", default=None)
args = ap.parse_args()
run_dir = Path(args.run_dir)
out = Path(args.out) if args.out else run_dir / "fusion_head_comparison_panel.png"
# load all images
roc_imgs = {(r, c): (mpimg.imread(str(run_dir / rel)), lbl)
for r, c, rel, lbl in ROC_CELLS}
expl_imgs = [(mpimg.imread(str(run_dir / rel)), lbl)
for _, rel, lbl in EXPL_ROWS]
# 4-row grid: rows 0-1 are the 2×2 ROC square; rows 2-3 are full-width explainability
fig = plt.figure(figsize=(14, 20))
gs = fig.add_gridspec(
4, 2,
height_ratios=[1, 1, 0.6, 0.6],
hspace=0.06,
wspace=0.04,
)
# ROC cells (2×2)
for row, col, _, _ in ROC_CELLS:
ax = fig.add_subplot(gs[row, col])
img, label = roc_imgs[(row, col)]
ax.imshow(img)
ax.axis("off")
ax.set_title(label, fontsize=11, pad=5)
# Explainability rows (span both columns)
for grid_row, (img, label) in zip([2, 3], expl_imgs):
ax = fig.add_subplot(gs[grid_row, :])
ax.imshow(img)
ax.axis("off")
ax.set_title(label, fontsize=11, pad=5)
out.parent.mkdir(parents=True, exist_ok=True)
fig.savefig(out, dpi=150, bbox_inches="tight")
plt.close(fig)
print(f"Saved → {out}")
if __name__ == "__main__":
main()
scripts/output_analysis/visualizations/build_md_explainability_panel.py
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env python3
"""
Assemble a metadata-explainability comparison panel.
Layout (2 rows × 4 cols):
row 0 = binary, row 1 = multiclass
col 0: single md_importance
col 1: ensemble md_importance
col 2: nocrop ROC
col 3: excl_phakic_axial ROC
Usage
-----
python scripts/output_analysis/visualizations/build_md_explainability_panel.py \
--nocrop-dir analysis_data/pipeline_nocrop \
--excl-dir analysis_data/pipeline_nocrop_excl_phakic_axial \
--out analysis_data/pipeline_nocrop/md_explainability_panel.png
"""
from __future__ import annotations
import argparse
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
# (row, col, dir_key, rel_path, label)
# dir_key: "nocrop" or "excl"
CELLS = [
# ---- binary row (row 0) ----
(0, 0, "nocrop", "binary/single/explainability_md_importance_summary.png",
"Binary — Single"),
(0, 1, "nocrop", "binary/ensemble/explainability_md_importance_summary.png",
"Binary — Ensemble"),
(0, 2, "nocrop", "binary/ensemble/plots/roc_probs_fused_mean_ovr.png",
"Binary — nocrop ROC"),
(0, 3, "excl", "binary/ensemble/plots/roc_probs_fused_mean_ovr.png",
"Binary — excl phakic+axial ROC"),
# ---- multiclass row (row 1) ----
(1, 0, "nocrop", "multiclass/single/explainability_md_importance_summary.png",
"Multiclass — Single"),
(1, 1, "nocrop", "multiclass/ensemble/explainability_md_importance_summary.png",
"Multiclass — Ensemble"),
(1, 2, "nocrop", "multiclass/ensemble/plots/roc_probs_fused_mean_ovr.png",
"Multiclass — nocrop ROC"),
(1, 3, "excl", "multiclass/ensemble/plots/roc_probs_fused_mean_ovr.png",
"Multiclass — excl phakic+axial ROC"),
]
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--nocrop-dir", default="analysis_data/pipeline_nocrop")
ap.add_argument("--excl-dir", default="analysis_data/pipeline_nocrop_excl_phakic_axial")
ap.add_argument("--out", default=None)
args = ap.parse_args()
nocrop_dir = Path(args.nocrop_dir)
excl_dir = Path(args.excl_dir)
out = Path(args.out) if args.out else nocrop_dir / "md_explainability_panel.png"
fig = plt.figure(figsize=(24, 12))
gs = fig.add_gridspec(
2, 4,
hspace=0.08,
wspace=0.04,
)
dirs = {"nocrop": nocrop_dir, "excl": excl_dir}
for row, col, dir_key, rel, label in CELLS:
ax = fig.add_subplot(gs[row, col])
img = mpimg.imread(str(dirs[dir_key] / rel))
ax.imshow(img)
ax.axis("off")
ax.set_title(label, fontsize=11, pad=5)
out.parent.mkdir(parents=True, exist_ok=True)
fig.savefig(out, dpi=150, bbox_inches="tight")
plt.close(fig)
print(f"Saved → {out}")
if __name__ == "__main__":
main()
scripts/output_analysis/visualizations/plot_early_stopping_sweep.py
+217
View File
@@ -0,0 +1,217 @@
#!/usr/bin/env python3
"""
Simulate early stopping at each epoch N and show what val/holdout AUC
you would have gotten if you stopped there.
For each fold and each candidate stopping epoch N:
- Find the epoch <= N with the highest val AUC (checkpoint selection)
- Record the val AUC and holdout AUC at that epoch
Then plot mean ± std across all folds as a function of N.
Usage
-----
python scripts/output_analysis/visualizations/plot_early_stopping_sweep.py \
--run-dir analysis_data/pipeline_10x5 \
--eval-mode binary \
--tower-mode ensemble \
--head fused
"""
from __future__ import annotations
import argparse
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
HEAD_COL = {
"fused": ("ensemble_val_auc", "ensemble_holdout_auc"),
"img": ("ensemble_val_auc_img", "ensemble_holdout_auc_img"),
"md": ("ensemble_val_auc_md", "ensemble_holdout_auc_md"),
"classic": ("classic_val_auc", "classic_holdout_auc"),
}
def load_fold_logs(run_dir: Path, eval_mode: str, tower_mode: str,
val_col: str, hld_col: str):
logs = []
for rep_dir in sorted(run_dir.glob("rep*")):
mode_dir = rep_dir / eval_mode / tower_mode
if not mode_dir.exists():
continue
for fd in sorted(
[d for d in mode_dir.iterdir() if d.is_dir() and d.name.startswith("fold")],
key=lambda p: int(p.name.replace("fold", "")),
):
log = fd / "epoch_log.csv"
if not log.exists():
continue
df = pd.read_csv(log)
if val_col not in df.columns or hld_col not in df.columns:
continue
df = df[["epoch", val_col, hld_col]].dropna()
logs.append(df.reset_index(drop=True))
return logs
def sweep(logs: list[pd.DataFrame], val_col: str, hld_col: str):
max_epoch = max(df["epoch"].max() for df in logs)
epochs = np.arange(1, int(max_epoch) + 1)
val_mat = np.full((len(logs), len(epochs)), np.nan)
hld_mat = np.full((len(logs), len(epochs)), np.nan)
for i, df in enumerate(logs):
for j, n in enumerate(epochs):
window = df[df["epoch"] <= n]
if window.empty:
continue
best_idx = window[val_col].idxmax()
val_mat[i, j] = window.loc[best_idx, val_col]
hld_mat[i, j] = window.loc[best_idx, hld_col]
return epochs, val_mat, hld_mat
def plot(epochs, val_mat, hld_mat, out_path: Path, title: str, inverted: bool = False, ci_tol: float = 0.0):
val_mean = np.nanmean(val_mat, axis=0)
val_std = np.nanstd(val_mat, axis=0)
hld_mean = np.nanmean(hld_mat, axis=0)
hld_std = np.nanstd(hld_mat, axis=0)
if inverted:
# compute cost per fold, then aggregate — avoids max-of-mean bias
best_val_per_fold = np.nanmax(val_mat, axis=1, keepdims=True) # (n_folds, 1)
best_hld_per_fold = np.nanmax(hld_mat, axis=1, keepdims=True)
delta_val = best_val_per_fold - val_mat # (n_folds, n_epochs)
delta_hld = best_hld_per_fold - hld_mat
y_val = np.nanmean(delta_val, axis=0)
y_hld = np.nanmean(delta_hld, axis=0)
sy_val = np.nanstd(delta_val, axis=0)
sy_hld = np.nanstd(delta_hld, axis=0)
else:
y_val, y_hld = val_mean, hld_mean
sy_val, sy_hld = val_std, hld_std
fig, ax = plt.subplots(figsize=(11, 5))
if inverted:
# faint per-fold lines
for i in range(delta_val.shape[0]):
ax.plot(epochs, delta_val[i], color="steelblue", linewidth=0.6, alpha=0.18)
ax.plot(epochs, delta_hld[i], color="firebrick", linewidth=0.6, alpha=0.18)
ax.plot(epochs, y_val, color="steelblue", linewidth=2.0,
label="Best val − val@N (val cost of stopping early)" if inverted
else "Val AUC (best ckpt up to N)")
ax.fill_between(epochs, y_val - sy_val, y_val + sy_val, color="steelblue", alpha=0.15)
ax.plot(epochs, y_hld, color="firebrick", linewidth=2.0,
label="Best hld − hld@N (holdout cost of stopping early)" if inverted
else "Holdout AUC (at best val ckpt)")
ax.fill_between(epochs, y_hld - sy_hld, y_hld + sy_hld, color="firebrick", alpha=0.15)
if inverted:
ax.axhline(0, color="black", linewidth=1.0, linestyle="--", alpha=0.4)
# CI-crosses-zero regions (with optional tolerance)
val_ci_zero = (y_val - sy_val) <= ci_tol
hld_ci_zero = (y_hld - sy_hld) <= ci_tol
both_ci_zero = val_ci_zero & hld_ci_zero
ymax = max(np.nanmax(y_val), np.nanmax(y_hld)) * 1.15
ax.fill_between(epochs, 0, ymax, where=val_ci_zero,
color="steelblue", alpha=0.12, label="val CI ≤ 0")
ax.fill_between(epochs, 0, ymax, where=hld_ci_zero,
color="firebrick", alpha=0.12, label="holdout CI ≤ 0")
ax.fill_between(epochs, 0, ymax, where=both_ci_zero,
color="purple", alpha=0.20, label="both CI ≤ 0")
ax.set_ylabel("AUC lost vs best achievable")
ax.set_ylim(-0.05, ymax)
legend_loc = "upper right"
else:
# gap curve on twin axis
gap_mean = val_mean - hld_mean
ax2 = ax.twinx()
ax2.plot(epochs, gap_mean, color="darkorange", linewidth=1.5,
linestyle="--", alpha=0.7, label="Val−Holdout gap")
ax2.set_ylabel("Val − Holdout gap", color="darkorange", fontsize=9)
ax2.tick_params(axis="y", labelcolor="darkorange")
ax2.set_ylim(-0.1, 0.4)
lines2, labels2 = ax2.get_legend_handles_labels()
best_hld_ep = epochs[np.nanargmax(hld_mean)]
best_hld_val = hld_mean[np.nanargmax(hld_mean)]
ax.axvline(best_hld_ep, color="firebrick", linewidth=1.2, linestyle=":",
alpha=0.8, label=f"peak holdout @ epoch {best_hld_ep} ({best_hld_val:.3f})")
stable = epochs >= 3
min_gap_ep = epochs[stable][np.nanargmin(gap_mean[stable])]
ax.axvline(min_gap_ep, color="darkorange", linewidth=1.2, linestyle=":",
alpha=0.8, label=f"min gap @ epoch {min_gap_ep}")
ax.set_ylabel("AUC")
ax.set_ylim(0.5, 1.05)
legend_loc = "lower right"
ax.set_xlabel("Stopping epoch N")
ax.set_title(title)
ax.grid(axis="y", linestyle="--", alpha=0.35)
lines1, labels1 = ax.get_legend_handles_labels()
if not inverted:
lines1 += lines2; labels1 += labels2
ax.legend(lines1, labels1, fontsize=8, loc=legend_loc)
out_path.parent.mkdir(parents=True, exist_ok=True)
fig.tight_layout()
fig.savefig(out_path, dpi=150)
plt.close(fig)
print(f"Saved → {out_path}")
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--run-dir", default="analysis_data/pipeline_10x5")
ap.add_argument("--eval-mode", default="binary")
ap.add_argument("--tower-mode", default="ensemble")
ap.add_argument("--head", default="fused", choices=list(HEAD_COL))
ap.add_argument("--out", default=None)
ap.add_argument("--inverted", action="store_true",
help="Plot best-achievable minus current (cost of stopping early)")
ap.add_argument("--ci-tol", type=float, default=0.0,
help="Tolerance for CI-crosses-zero shading (default 0.0)")
args = ap.parse_args()
run_dir = Path(args.run_dir)
val_col, hld_col = HEAD_COL[args.head]
logs = load_fold_logs(run_dir, args.eval_mode, args.tower_mode, val_col, hld_col)
if not logs:
print("No epoch_log.csv files found.")
return
print(f"Loaded {len(logs)} fold logs")
epochs, val_mat, hld_mat = sweep(logs, val_col, hld_col)
tol_tag = f"_tol{args.ci_tol}" if args.ci_tol else ""
suffix = f"_inverted{tol_tag}" if args.inverted else ""
out = Path(args.out) if args.out else (
run_dir / args.eval_mode / args.tower_mode / "plots" /
f"early_stopping_sweep_{args.head}{suffix}.png"
)
title = ("Simulated early stopping — cost of stopping at epoch N\n"
if args.inverted else
"Simulated early stopping sweep\n")
title += f"{run_dir.name} · {args.eval_mode}/{args.tower_mode} · head={args.head}"
if args.inverted and args.ci_tol:
title += f" (CI tol={args.ci_tol})"
plot(epochs, val_mat, hld_mat, out, title, inverted=args.inverted, ci_tol=args.ci_tol)
if __name__ == "__main__":
main()
scripts/output_analysis/visualizations/plot_holdout_epoch_curves.py
+176
View File
@@ -0,0 +1,176 @@
#!/usr/bin/env python3
"""
Plot per-epoch holdout metrics across all folds in a 10x5 (or any multi-rep) run.
Each fold gets its own line. Lines are coloured by rep.
Modes
-----
holdout — raw holdout AUC per epoch (original plot)
delta — val_auc - holdout_auc per epoch (generalization gap;
closer to 0 = val most faithfully reflects holdout)
Usage
-----
python scripts/output_analysis/visualizations/plot_holdout_epoch_curves.py \
--run-dir analysis_data/pipeline_10x5 \
--eval-mode binary \
--tower-mode ensemble \
--head fused \
--mode delta
"""
from __future__ import annotations
import argparse
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
HEAD_COL = {
"fused": ("ensemble_val_auc", "ensemble_holdout_auc"),
"img": ("ensemble_val_auc_img", "ensemble_holdout_auc_img"),
"md": ("ensemble_val_auc_md", "ensemble_holdout_auc_md"),
"classic": ("classic_val_auc", "classic_holdout_auc"),
}
def load_curves(run_dir: Path, eval_mode: str, tower_mode: str,
val_col: str, hld_col: str, mode: str):
"""
Returns list of (rep, fold, epochs_array, values_array).
mode='holdout' → values = holdout_auc
mode='delta' → values = val_auc - holdout_auc
"""
curves = []
for rep_dir in sorted(run_dir.glob("rep*")):
mode_dir = rep_dir / eval_mode / tower_mode
if not mode_dir.exists():
continue
fold_dirs = sorted(
[d for d in mode_dir.iterdir() if d.is_dir() and d.name.startswith("fold")],
key=lambda p: int(p.name.replace("fold", "")),
)
for fd in fold_dirs:
log = fd / "epoch_log.csv"
if not log.exists():
continue
df = pd.read_csv(log)
needed = [hld_col] if mode == "holdout" else [val_col, hld_col]
if any(c not in df.columns for c in needed):
continue
df = df.dropna(subset=needed)
if mode == "holdout":
values = df[hld_col].to_numpy()
elif mode == "delta":
values = (df[val_col] - df[hld_col]).to_numpy()
else: # delta_adj
val_arr = df[val_col].to_numpy()
hld_arr = df[hld_col].to_numpy()
val_best = np.nanmax(val_arr)
penalty = val_best - val_arr # 0 when val is at its peak
values = (val_arr - hld_arr) + penalty
curves.append((rep_dir.name, fd.name, df["epoch"].to_numpy(), values))
return curves
def _build_mean_matrix(curves):
all_ep = max(len(e) for _, _, e, _ in curves)
mat = np.full((len(curves), all_ep), np.nan)
for i, (_, _, e, a) in enumerate(curves):
mat[i, :len(a)] = a
return mat, np.arange(1, all_ep + 1)
def plot(curves, mode: str, out_path: Path, title: str):
reps = sorted(set(r for r, _, _, _ in curves))
cmap = matplotlib.colormaps.get_cmap("tab10")
rep_color = {r: cmap(i / max(len(reps) - 1, 1)) for i, r in enumerate(reps)}
fig, ax = plt.subplots(figsize=(12, 6))
for rep, fold, epochs, vals in curves:
ax.plot(epochs, vals, color=rep_color[rep], alpha=0.3, linewidth=0.9)
# per-rep mean
for rep in reps:
rep_curves = [(e, a) for r, _, e, a in curves if r == rep]
max_ep = max(len(e) for e, _ in rep_curves)
mat = np.full((len(rep_curves), max_ep), np.nan)
for i, (e, a) in enumerate(rep_curves):
mat[i, :len(a)] = a
mean_curve = np.nanmean(mat, axis=0)
ax.plot(np.arange(1, max_ep + 1), mean_curve,
color=rep_color[rep], linewidth=1.8, alpha=0.85, label=rep)
# global mean ± std
all_mat, ep_axis = _build_mean_matrix(curves)
global_mean = np.nanmean(all_mat, axis=0)
global_std = np.nanstd(all_mat, axis=0)
ax.plot(ep_axis, global_mean, color="black", linewidth=2.5, zorder=5, label="global mean")
ax.fill_between(ep_axis, global_mean - global_std, global_mean + global_std,
color="black", alpha=0.12, zorder=4)
if mode in ("delta", "delta_adj"):
ax.axhline(0, color="black", linewidth=1.0, linestyle="--", alpha=0.5)
if mode == "delta":
ax.set_ylabel("Val AUC − Holdout AUC (gap)")
else:
ax.set_ylabel("(Val − Holdout) + (ValBest − Val) (adjusted gap)")
min_ep = int(ep_axis[np.nanargmin(global_mean)])
min_val = global_mean[np.nanargmin(global_mean)]
ax.axvline(min_ep, color="red", linewidth=1.2, linestyle=":", alpha=0.7,
label=f"min adjusted gap @ epoch {min_ep} ({min_val:+.3f})")
else:
ax.set_ylabel("Holdout AUC")
ax.set_ylim(0, 1.05)
ax.set_xlabel("Epoch")
ax.set_title(title)
ax.legend(fontsize=7, ncol=2, loc="upper right" if mode == "delta" else "lower right")
ax.grid(axis="y", linestyle="--", alpha=0.4)
out_path.parent.mkdir(parents=True, exist_ok=True)
fig.tight_layout()
fig.savefig(out_path, dpi=150)
plt.close(fig)
print(f"Saved → {out_path}")
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--run-dir", default="analysis_data/pipeline_10x5")
ap.add_argument("--eval-mode", default="binary")
ap.add_argument("--tower-mode", default="ensemble")
ap.add_argument("--head", default="fused", choices=list(HEAD_COL))
ap.add_argument("--mode", default="holdout", choices=["holdout", "delta", "delta_adj"])
ap.add_argument("--out", default=None)
args = ap.parse_args()
run_dir = Path(args.run_dir)
val_col, hld_col = HEAD_COL[args.head]
curves = load_curves(run_dir, args.eval_mode, args.tower_mode,
val_col, hld_col, args.mode)
if not curves:
print("No epoch_log.csv files found — check --run-dir / --eval-mode / --tower-mode")
return
print(f"Loaded {len(curves)} fold curves, up to {max(len(e) for _,_,e,_ in curves)} epochs each")
out = Path(args.out) if args.out else (
run_dir / args.eval_mode / args.tower_mode / "plots" /
f"holdout_epoch_curves_{args.head}_{args.mode}.png"
)
label = {"holdout": "holdout AUC", "delta": "val−holdout gap", "delta_adj": "val−holdout gap (val-adjusted)"}[args.mode]
title = (f"Per-fold {label} by epoch\n"
f"{run_dir.name} · {args.eval_mode}/{args.tower_mode} · head={args.head}")
plot(curves, args.mode, out, title)
if __name__ == "__main__":
main()
scripts/output_analysis/visualizations/plot_mdonly_curves.py
+167
View File
@@ -0,0 +1,167 @@
#!/usr/bin/env python3
"""
Per-epoch learning curves for mdonly runs.
Reads epoch_log.csv from each fold dir and plots val_auc, val_acc,
hld_auc, hld_acc — one figure per metric, all folds as individual lines.
Usage:
python scripts/output_analysis/visualizations/plot_mdonly_curves.py \
--run-dirs analysis_data/pipeline_mdonly_50ep \
analysis_data/pipeline_mdonly_200ep \
analysis_data/pipeline_mdonly_500ep \
--eval-mode binary
"""
from __future__ import annotations
import argparse
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
METRICS = [
("val_auc", "Val AUC"),
("val_acc", "Val Accuracy"),
("hld_auc", "Holdout AUC"),
("hld_acc", "Holdout Accuracy"),
]
PHASE_SHADING = {
"tower_warmup": "#d0e8ff",
"fused_warmup": "#d0ffe8",
}
def _load_folds(mode_dir: Path) -> list[pd.DataFrame]:
fold_dirs = sorted(
[p for p in mode_dir.glob("fold*") if p.is_dir()],
key=lambda p: int(p.name.replace("fold", "")),
)
frames = []
for fd in fold_dirs:
csv = fd / "epoch_log.csv"
if not csv.exists():
print(f" [warn] {csv} not found, skipping")
continue
df = pd.read_csv(csv)
df["_fold"] = int(fd.name.replace("fold", ""))
frames.append(df)
return frames
def _shade_warmup(ax: plt.Axes, df: pd.DataFrame) -> None:
"""Shade warmup phase regions based on first fold's phase column."""
if "phase" not in df.columns:
return
prev_phase = None
start = None
for _, row in df.iterrows():
phase = row["phase"]
ep = row["epoch"]
if phase != prev_phase:
if prev_phase in PHASE_SHADING and start is not None:
ax.axvspan(start - 0.5, ep - 0.5, color=PHASE_SHADING[prev_phase],
alpha=0.35, zorder=0, label=f"{prev_phase.replace('_', ' ')}")
start = ep
prev_phase = phase
# close last span
if prev_phase in PHASE_SHADING and start is not None:
ax.axvspan(start - 0.5, df["epoch"].max() + 0.5,
color=PHASE_SHADING[prev_phase], alpha=0.35, zorder=0)
def plot_curves(
run_dirs: list[Path],
eval_mode: str,
tower_mode: str,
out_dir: Path | None,
) -> None:
# Collect (label, frames) pairs
datasets: list[tuple[str, list[pd.DataFrame]]] = []
for rd in run_dirs:
mode_dir = rd / eval_mode / tower_mode
if not mode_dir.exists():
print(f" [skip] {mode_dir} not found")
continue
frames = _load_folds(mode_dir)
if not frames:
print(f" [skip] no epoch_log.csv found under {mode_dir}")
continue
datasets.append((rd.name, frames))
if not datasets:
print("No data found — nothing to plot.")
return
# One figure per metric
for metric_key, metric_label in METRICS:
# Check any fold actually has this metric with non-nan values
has_data = any(
not frames[0][metric_key].isna().all()
for _, frames in datasets
if frames and metric_key in frames[0].columns
)
if not has_data:
continue
n_runs = len(datasets)
fig, axes = plt.subplots(1, n_runs, figsize=(5 * n_runs, 4.5), squeeze=False)
for col_idx, (run_label, frames) in enumerate(datasets):
ax = axes[0][col_idx]
if frames and metric_key in frames[0].columns:
_shade_warmup(ax, frames[0])
colours = plt.cm.tab10(np.linspace(0, 0.9, len(frames)))
for frame, colour in zip(frames, colours):
if metric_key not in frame.columns:
continue
vals = frame[metric_key].values
epochs = frame["epoch"].values
mask = ~np.isnan(vals.astype(float))
if mask.sum() == 0:
continue
ax.plot(epochs[mask], vals[mask],
linewidth=1.4, color=colour,
label=f"fold {frame['_fold'].iloc[0]}")
ax.set_title(run_label, fontsize=10)
ax.set_xlabel("Epoch")
if col_idx == 0:
ax.set_ylabel(metric_label)
ax.legend(fontsize=7, loc="lower right")
ax.grid(True, linewidth=0.4, alpha=0.5)
fig.suptitle(f"{metric_label} [{eval_mode} / {tower_mode}]", fontsize=12)
fig.tight_layout()
dest = out_dir or (run_dirs[0].parent / "mdonly_plots")
dest.mkdir(parents=True, exist_ok=True)
fname = f"mdonly_{metric_key}_{eval_mode}_{tower_mode}.png"
fig.savefig(dest / fname, dpi=150, bbox_inches="tight")
plt.close(fig)
print(f"Saved: {dest / fname}")
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--run-dirs", nargs="+", required=True,
help="One or more run directories (e.g. analysis_data/pipeline_mdonly_50ep).")
ap.add_argument("--eval-mode", default="binary", choices=["binary", "multiclass"])
ap.add_argument("--tower-mode", default="single", choices=["single", "ensemble"])
ap.add_argument("--out", default=None,
help="Output directory for plots (default: {first_run_dir}/../mdonly_plots).")
args = ap.parse_args()
run_dirs = [Path(d) for d in args.run_dirs]
out_dir = Path(args.out) if args.out else None
plot_curves(run_dirs, args.eval_mode, args.tower_mode, out_dir)
if __name__ == "__main__":
main()
scripts/output_analysis/visualizations/plot_prob_strips.py
+714
View File
@@ -0,0 +1,714 @@
"""
Plot predicted-probability strip charts for V2 runs.
Three styles available via --style:
strips (default)
X = true class, Y = P(Glaucoma), color = true class × fold shade.
Works for binary and multiclass.
confidence
X = predicted class (major) subdivided by true class (minor sub-column).
Y = model confidence in its own prediction (P of the predicted class).
Color = true class × fold shade.
Makes high-confidence mistakes immediately visible.
triangle (multiclass only)
Ternary / simplex plot. Each corner = 100% probability for one class.
Every sample is a dot placed at its softmax probability vector
(p_H, p_G, p_S) using barycentric coordinates. The centroid is maximum
uncertainty (1/3, 1/3, 1/3). Correctly classified samples cluster near
their true-class corner; mistakes drift toward the wrong corner.
Colour families (light → dark = fold 0 → fold N-1):
Blues = Healthy / Normal eyes
Reds = Glaucoma eyes
Greens = Suspect eyes
Usage
-----
python scripts/output_analysis/visualizations/plot_prob_strips.py \
--run-dir analysis_data/v2.3_single_multiclass_nocrop/multiclass/single \
--style triangle --head fused
"""
from __future__ import annotations
import argparse
import re
from pathlib import Path
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import numpy as np
import pandas as pd
# ── colour families ────────────────────────────────────────────────────────────
_CLASS_FAMILIES = {
0: ("#aac4ff", "#0a3d91"), # blues (healthy / normal)
1: ("#ffaaaa", "#8b0000"), # reds (glaucoma)
2: ("#aaffcc", "#1a6b3c"), # greens (suspect)
}
_CLASS_LABELS = {
0: "Healthy",
1: "Glaucoma",
2: "Suspect",
}
_CLASS_LABELS_SHORT = {
0: "H",
1: "G",
2: "S",
}
def _lerp_hex(c1: str, c2: str, t: float) -> tuple:
def h(c): return tuple(int(c.lstrip("#")[i*2:i*2+2], 16) / 255 for i in range(3))
r1, g1, b1 = h(c1)
r2, g2, b2 = h(c2)
return (r1 + t*(r2-r1), g1 + t*(g2-g1), b1 + t*(b2-b1))
def _fold_colours(n_folds: int) -> dict[int, dict[int, tuple]]:
out: dict[int, dict[int, tuple]] = {}
for cls, (light, dark) in _CLASS_FAMILIES.items():
out[cls] = {}
for f in range(n_folds):
t = f / max(n_folds - 1, 1)
out[cls][f] = _lerp_hex(light, dark, t)
return out
def _fold_dirs(run_dir: Path) -> list[Path]:
return sorted(
[p for p in run_dir.glob("fold*") if p.is_dir() and re.search(r"\d+", p.name)],
key=lambda p: int(re.search(r"\d+", p.name).group()),
)
def _detect_heads(run_dir: Path) -> list[str]:
"""Return all head names available in the first non-empty fold dir."""
for fd in _fold_dirs(run_dir):
found: list[str] = []
# Standard heads come from the predictions CSV — prefer per-eye
csv = fd / "predictions_pereye.csv"
if not csv.exists():
csv = fd / "predictions_classic.csv"
if not csv.exists():
csv = fd / "predictions.csv"
if csv.exists():
cols = pd.read_csv(csv, nrows=0).columns.tolist()
for h in ["fused", "img", "md"]:
if f"prob_{h}_c0" in cols:
found.append(h)
# Fusion head lives in a separate npy
if (fd / "probs_fused_head.npy").exists():
found.append("fused_head")
if found:
return found
return ["fused"] # safe fallback
def _load_folds(run_dir: Path, head: str) -> list[pd.DataFrame]:
dirs = _fold_dirs(run_dir)
if not dirs:
raise FileNotFoundError(f"No fold* directories found under {run_dir}")
frames = []
for fd in dirs:
fold_num = int(re.search(r"\d+", fd.name).group())
# fused_head is stored as npy, not in the predictions CSV
if head == "fused_head":
y_path = fd / "y_true.npy"
p_path = fd / "probs_fused_head.npy"
if not y_path.exists() or not p_path.exists():
print(f" [warn] fused_head npy not found in {fd}, skipping")
continue
y = np.load(y_path)
p = np.load(p_path)
df = pd.DataFrame({"y_true": y})
for c in range(p.shape[1]):
df[f"prob_fused_head_c{c}"] = p[:, c]
df["_fold"] = fold_num
frames.append(df)
continue
# Standard heads from predictions CSV — prefer per-eye (2× dots, no OD/OS averaging)
csv = fd / "predictions_pereye.csv"
if not csv.exists():
csv = fd / "predictions_classic.csv"
if not csv.exists():
csv = fd / "predictions.csv"
if not csv.exists():
print(f" [warn] no predictions CSV found in {fd}, skipping")
continue
df = pd.read_csv(csv)
df["_fold"] = fold_num
frames.append(df)
if not frames:
raise FileNotFoundError(
f"No data found for head='{head}' in any fold dir under {run_dir}"
)
return frames
def _detect_num_classes(df: pd.DataFrame, head: str) -> int:
return len([c for c in df.columns if c.startswith(f"prob_{head}_c")])
def _draw_grid_legend(ax_leg: plt.Axes, n_folds: int, classes: list[int],
colours: dict, title: str = "True class") -> None:
"""Rows = folds, columns = classes grid of coloured dots."""
from matplotlib.patches import FancyBboxPatch
ax_leg.axis("off")
col_xs = np.linspace(0.55, 0.88, len(classes)) if len(classes) > 1 else [0.72]
row_ys = np.linspace(0.88, 0.05, n_folds + 1)
header_y, dot_ys = row_ys[0], row_ys[1:]
for ci, cls in enumerate(classes):
ax_leg.text(col_xs[ci], header_y, _CLASS_LABELS_SHORT.get(cls, f"C{cls}"),
ha="center", va="bottom", fontsize=8, fontweight="bold",
transform=ax_leg.transAxes)
for fi in range(n_folds):
y = dot_ys[fi]
ax_leg.text(0.05, y, f"Fold {fi}", ha="left", va="center", fontsize=9,
transform=ax_leg.transAxes)
for ci, cls in enumerate(classes):
ax_leg.scatter([col_xs[ci]], [y], color=colours[cls][fi], s=70, zorder=3,
transform=ax_leg.transAxes, clip_on=False)
ax_leg.add_patch(FancyBboxPatch((0, 0), 1, 1, boxstyle="round,pad=0.02",
linewidth=0.8, edgecolor="#aaaaaa",
facecolor="#f9f9f9", zorder=0,
transform=ax_leg.transAxes))
ax_leg.set_title(title, fontsize=9, pad=4)
# ── strips style ───────────────────────────────────────────────────────────────
def plot_strip(run_dir: Path, head: str = "fused", out_dir: Path | None = None,
jitter_strength: float = 0.08) -> Path:
"""X = true class, Y = P(Glaucoma)."""
frames = _load_folds(run_dir, head)
all_true = sorted({int(v) for df in frames for v in df["y_true"]})
n_folds = len(frames)
colours = _fold_colours(n_folds)
rng = np.random.default_rng(seed=0)
prob_col = f"prob_{head}_c1"
x_pos = {cls: i for i, cls in enumerate(all_true)}
leg_width = 0.8 + 0.55 * len(all_true)
fig = plt.figure(figsize=(max(6, 2.2 * len(all_true) + 1), 7))
gs = fig.add_gridspec(1, 2, width_ratios=[max(6, 2.2 * len(all_true)), leg_width],
wspace=0.08)
ax = fig.add_subplot(gs[0])
ax_leg = fig.add_subplot(gs[1])
fig.suptitle(f"{run_dir.parent.parent.name} — P(Glaucoma) [{head}]",
fontsize=11, y=1.01)
for fold_idx, df in enumerate(frames):
for cls in all_true:
sub = df[df["y_true"] == cls]
if sub.empty:
continue
probs = sub[prob_col].values
jitter = rng.uniform(-jitter_strength, jitter_strength, size=len(probs))
ax.scatter(x_pos[cls] + jitter, probs, color=colours[cls][fold_idx],
s=45, alpha=0.88, linewidths=0, zorder=3)
ax.set_xticks(list(x_pos.values()))
ax.set_xticklabels([_CLASS_LABELS.get(c, f"C{c}") for c in all_true], fontsize=12)
ax.set_xlim(-0.5, len(all_true) - 0.5)
ax.set_ylim(-0.05, 1.05)
ax.set_ylabel("Predicted P(Glaucoma)", fontsize=12)
ax.set_xlabel("True class", fontsize=12)
ax.axhline(0.5, color="grey", linestyle="--", linewidth=0.8, alpha=0.5)
ax.grid(axis="y", linestyle=":", alpha=0.4)
_draw_grid_legend(ax_leg, n_folds, all_true, colours, title="Legend")
fig.tight_layout()
return _save(fig, out_dir or run_dir / "plots", f"prob_strips_{head}.png")
# ── confidence style ───────────────────────────────────────────────────────────
def plot_confidence(run_dir: Path, head: str = "fused", out_dir: Path | None = None,
jitter_strength: float = 0.06) -> Path:
"""
X = predicted class (major) × true class (minor sub-column).
Y = model confidence = P(predicted class).
Color = true class × fold shade.
"""
frames = _load_folds(run_dir, head)
num_cls = _detect_num_classes(frames[0], head)
all_cls = list(range(num_cls))
all_true = sorted({int(v) for df in frames for v in df["y_true"]})
n_folds = len(frames)
colours = _fold_colours(n_folds)
rng = np.random.default_rng(seed=0)
# Sub-column spacing within each predicted-class group
# E.g. for 3 classes: sub-offsets at -0.25, 0, +0.25
n_sub = len(all_true)
sub_spacing = 0.22
sub_offsets = np.linspace(-(n_sub - 1) * sub_spacing / 2,
(n_sub - 1) * sub_spacing / 2,
n_sub)
sub_off = {cls: sub_offsets[i] for i, cls in enumerate(all_true)}
# Major x positions for each predicted class, spaced so sub-columns don't bleed
group_gap = sub_spacing * n_sub + 0.35
major_x = {pc: i * group_gap for i, pc in enumerate(all_cls)}
leg_width = 0.8 + 0.55 * n_sub
fig_w = max(7, group_gap * len(all_cls) * 1.8 + 1)
fig = plt.figure(figsize=(fig_w, 7))
gs = fig.add_gridspec(1, 2, width_ratios=[fig_w - leg_width, leg_width],
wspace=0.08)
ax = fig.add_subplot(gs[0])
ax_leg = fig.add_subplot(gs[1])
fig.suptitle(f"{run_dir.parent.parent.name} — Prediction confidence [{head}]",
fontsize=11, y=1.01)
for fold_idx, df in enumerate(frames):
prob_cols = [f"prob_{head}_c{c}" for c in all_cls]
pred_col = f"pred_{head}"
for true_cls in all_true:
sub = df[df["y_true"] == true_cls].copy()
if sub.empty:
continue
for pred_cls in all_cls:
rows = sub[sub[pred_col] == pred_cls]
if rows.empty:
continue
# confidence = probability assigned to the predicted class
conf = rows[f"prob_{head}_c{pred_cls}"].values
x_base = major_x[pred_cls] + sub_off[true_cls]
jitter = rng.uniform(-jitter_strength * 0.5,
jitter_strength * 0.5, size=len(conf))
ax.scatter(x_base + jitter, conf, color=colours[true_cls][fold_idx],
s=45, alpha=0.88, linewidths=0, zorder=3)
# X-axis: major ticks with predicted-class labels, minor sub-column markers
ax.set_xlim(-group_gap * 0.5, group_gap * len(all_cls) - group_gap * 0.5)
ax.set_xticks([major_x[pc] for pc in all_cls])
ax.set_xticklabels([_CLASS_LABELS.get(pc, f"C{pc}") for pc in all_cls], fontsize=12)
# Light vertical separators between predicted-class groups
for i in range(1, len(all_cls)):
sep_x = (major_x[all_cls[i-1]] + major_x[all_cls[i]]) / 2
ax.axvline(sep_x, color="#cccccc", linewidth=1.0, zorder=1)
# Sub-column labels (H/G/S) just below the x-axis
for pred_cls in all_cls:
for true_cls in all_true:
lbl = _CLASS_LABELS_SHORT.get(true_cls, f"C{true_cls}")
ax.text(major_x[pred_cls] + sub_off[true_cls], -0.085, lbl,
ha="center", va="top", fontsize=7, color="#555555",
transform=ax.get_xaxis_transform())
ax.set_ylim(-0.05, 1.05)
ax.set_ylabel("Confidence P(predicted class)", fontsize=12)
ax.set_xlabel("Predicted class", fontsize=12, labelpad=18)
ax.axhline(0.5, color="grey", linestyle="--", linewidth=0.8, alpha=0.5)
ax.grid(axis="y", linestyle=":", alpha=0.4)
_draw_grid_legend(ax_leg, n_folds, all_true, colours, title="True class")
fig.tight_layout()
return _save(fig, out_dir or run_dir / "plots", f"prob_confidence_{head}.png")
# ── triangle / ternary style ───────────────────────────────────────────────────
# Equilateral triangle vertices in Cartesian space:
# H (Healthy) = bottom-left (0, 0)
# G (Glaucoma) = bottom-right (1, 0)
# S (Suspect) = apex (0.5, sqrt(3)/2)
_TRI_VERTICES = np.array([
[0.0, 0.0], # class 0 – Healthy
[1.0, 0.0], # class 1 – Glaucoma
[0.5, np.sqrt(3) / 2], # class 2 – Suspect
])
def _bary_to_cart(probs: np.ndarray) -> np.ndarray:
"""
Convert Nx3 barycentric coordinates (softmax probs) to Nx2 Cartesian.
probs rows must sum to 1.
"""
return probs @ _TRI_VERTICES
def _draw_triangle_grid(ax: plt.Axes, levels: tuple = (0.25, 0.5, 0.75)) -> None:
"""Draw the triangle border and iso-probability grid lines."""
from matplotlib.patches import Polygon
from matplotlib.lines import Line2D
# Outer triangle
tri = Polygon(_TRI_VERTICES, fill=False, edgecolor="#333333", linewidth=1.5, zorder=2)
ax.add_patch(tri)
# Grid lines: for each class, lines where p_class = level,
# parallel to the opposite edge.
for level in levels:
for cls in range(3):
# Points on the two edges adjacent to this vertex at distance `level`
v0 = _TRI_VERTICES[cls]
v1 = _TRI_VERTICES[(cls + 1) % 3]
v2 = _TRI_VERTICES[(cls + 2) % 3]
# A line at p_cls = level divides the triangle:
# p1 = level * v0 + (1-level) * v1
# p2 = level * v0 + (1-level) * v2
p1 = level * v0 + (1 - level) * v1
p2 = level * v0 + (1 - level) * v2
ax.plot([p1[0], p2[0]], [p1[1], p2[1]],
color="#cccccc", linewidth=0.6, zorder=1, linestyle="--")
def plot_triangle(run_dir: Path, head: str = "fused", out_dir: Path | None = None) -> Path:
"""
Ternary simplex plot of softmax probabilities for a 3-class run.
Each dot = one eye; position = (p_H, p_G, p_S) in barycentric coords.
Color = true class × fold shade.
"""
frames = _load_folds(run_dir, head)
num_cls = _detect_num_classes(frames[0], head)
if num_cls != 3:
raise ValueError(f"Triangle plot requires 3 classes; got {num_cls}. "
"Use --style strips for binary.")
all_true = sorted({int(v) for df in frames for v in df["y_true"]})
n_folds = len(frames)
colours = _fold_colours(n_folds)
leg_width = 0.8 + 0.55 * len(all_true)
fig = plt.figure(figsize=(8, 7))
gs = fig.add_gridspec(1, 2, width_ratios=[8 - leg_width, leg_width], wspace=0.05)
ax = fig.add_subplot(gs[0], aspect="equal")
ax_leg = fig.add_subplot(gs[1])
fig.suptitle(f"{run_dir.parent.parent.name} — Simplex [{head}]",
fontsize=11, y=1.01)
_draw_triangle_grid(ax)
# Vertex labels — placed just outside each corner
offsets = [(-0.07, -0.06), (0.07, -0.06), (0.0, 0.06)]
for cls_idx in range(3):
lbl = _CLASS_LABELS.get(cls_idx, f"C{cls_idx}")
vx, vy = _TRI_VERTICES[cls_idx]
dx, dy = offsets[cls_idx]
ax.text(vx + dx, vy + dy, lbl, ha="center", va="center",
fontsize=12, fontweight="bold")
# Centroid marker
cx, cy = _TRI_VERTICES.mean(axis=0)
ax.scatter([cx], [cy], color="#aaaaaa", s=30, marker="+", zorder=2, linewidths=1)
ax.text(cx + 0.02, cy - 0.04, "1/3 each", fontsize=7, color="#999999", ha="left")
# Data dots
rng = np.random.default_rng(seed=0)
for fold_idx, df in enumerate(frames):
prob_cols = [f"prob_{head}_c{c}" for c in range(3)]
probs_all = df[prob_cols].values # Nx3
y_true = df["y_true"].values.astype(int)
for true_cls in all_true:
mask = y_true == true_cls
if not mask.any():
continue
probs = probs_all[mask] # Kx3
xy = _bary_to_cart(probs) # Kx2
noise = rng.normal(0, 0.004, xy.shape)
ax.scatter(xy[:, 0] + noise[:, 0],
xy[:, 1] + noise[:, 1],
color=colours[true_cls][fold_idx],
s=30, alpha=0.80, linewidths=0, zorder=3)
ax.set_xlim(-0.18, 1.18)
ax.set_ylim(-0.12, 1.02)
ax.axis("off")
_draw_grid_legend(ax_leg, n_folds, all_true, colours, title="True class")
fig.tight_layout()
return _save(fig, out_dir or run_dir / "plots", f"prob_triangle_{head}.png")
# ── triangle3d style ──────────────────────────────────────────────────────────
# Triangle vertices for the 3D "bread-slice" view.
# Triangles stand upright in the x-z plane; y is the depth (layer) axis.
# H (Healthy) = bottom-left (0, 0) — back corner of the base
# G (Glaucoma) = top-centre (0.5, √3/2) — apex (visually "at the top")
# S (Suspect) = bottom-right (1, 0) — front corner toward the viewer
_TRI3D_VERTS = np.array([
[0.0, 0.0], # class 0 – Healthy (bottom-left / back)
[0.5, np.sqrt(3) / 2], # class 1 – Glaucoma (top)
[1.0, 0.0], # class 2 – Suspect (bottom-right / front)
])
def _bary_to_cart_3d(probs: np.ndarray) -> np.ndarray:
"""Convert Nx3 softmax probs to Nx2 Cartesian using the 3D vertex layout."""
return probs @ _TRI3D_VERTS
def plot_triangle_3d(
run_dir: Path,
head: str = "fused",
out_dir: Path | None = None,
elev: float = 20,
azim: float = -45,
y_spacing: float = 0.4,
) -> Path:
"""
3-D ternary "bread-slice" plot.
Each true class gets its own vertical triangle slice standing in the x-z
plane, stacked along the y (depth) axis (camera at ~4:30 / SE position):
• Healthy layer is at y=0 (front-left from 4:30 view)
• Glaucoma layer is at y=1
• Suspect layer is at y=2 (back-right from 4:30 view)
Within every slice the simplex corners are:
• G (Glaucoma) at the top apex
• H (Healthy) at the bottom-left (back corner of the base)
• S (Suspect) at the bottom-right (front corner toward viewer)
View with elev/azim to see the slices as near-vertical planes with slight
perspective depth.
"""
from mpl_toolkits.mplot3d import Axes3D # noqa: F401
from mpl_toolkits.mplot3d.art3d import Poly3DCollection
from matplotlib.patches import Patch
frames = _load_folds(run_dir, head)
num_cls = _detect_num_classes(frames[0], head)
if num_cls != 3:
raise ValueError("triangle3d requires 3 classes.")
all_true = sorted({int(v) for df in frames for v in df["y_true"]})
n_folds = len(frames)
colours = _fold_colours(n_folds)
rng = np.random.default_rng(seed=0)
# Layer y positions: H front-left (y=0), S back-right (y=2) from 4:30 view
y_pos = {cls: i * y_spacing for i, cls in enumerate(all_true)}
# Triangle wireframe in (x, z) — loop closed
wire_x = np.append(_TRI3D_VERTS[:, 0], _TRI3D_VERTS[0, 0])
wire_z = np.append(_TRI3D_VERTS[:, 1], _TRI3D_VERTS[0, 1])
fig = plt.figure(figsize=(9, 7))
ax = fig.add_subplot(111, projection="3d")
ax.view_init(elev=elev, azim=azim)
fig.suptitle(run_dir.parent.parent.name, fontsize=11)
# ── pre-compute all xz points per class (needed for KDE) ─────────────────
from scipy.stats import gaussian_kde
import matplotlib.colors as mcolors
_xz_by_cls: dict[int, np.ndarray] = {}
for df in frames:
prob_cols_all = [f"prob_{head}_c{c}" for c in range(3)]
p_all = df[prob_cols_all].values
yt_all = df["y_true"].values.astype(int)
for cls in all_true:
m = yt_all == cls
if m.any():
xz_pts = _bary_to_cart_3d(p_all[m])
_xz_by_cls[cls] = (
np.vstack([_xz_by_cls[cls], xz_pts])
if cls in _xz_by_cls else xz_pts
)
# KDE grid setup — evaluate on a 40×40 grid masked to the triangle interior
_G = 40
_x_lin = np.linspace(0.0, 1.0, _G)
_z_lin = np.linspace(0.0, np.sqrt(3) / 2, _G)
_dx = _x_lin[1] - _x_lin[0]
_dz = _z_lin[1] - _z_lin[0]
_XX, _ZZ = np.meshgrid(_x_lin, _z_lin) # (_G, _G)
# Vectorised inside-triangle test (barycentric)
vH, vG, vS = _TRI3D_VERTS
_denom = (vG[1] - vS[1]) * (vH[0] - vS[0]) + (vS[0] - vG[0]) * (vH[1] - vS[1])
def _inside(px, pz):
la = ((vG[1] - vS[1]) * (px - vS[0]) + (vS[0] - vG[0]) * (pz - vS[1])) / _denom
lb = ((vS[1] - vH[1]) * (px - vS[0]) + (vH[0] - vS[0]) * (pz - vS[1])) / _denom
return (la >= 0) & (lb >= 0) & ((1 - la - lb) >= 0)
_inside_mask = _inside(_XX.ravel(), _ZZ.ravel()).reshape(_G, _G)
_grid_pts = np.vstack([_XX.ravel(), _ZZ.ravel()]) # 2×(_G²)
# ── draw triangle wireframe + density heatmap at each y layer ─────────────
for cls in all_true:
y = y_pos[cls]
base_col = colours[cls][n_folds // 2]
rgba_base = np.array(mcolors.to_rgba(base_col))
# Wireframe
ax.plot(wire_x, np.full_like(wire_x, y), wire_z,
color=base_col, linewidth=1.2, alpha=0.65, zorder=1)
# ── density heatmap ────────────────────────────────────────────────
xz_pts = _xz_by_cls.get(cls)
if xz_pts is not None and len(xz_pts) >= 2:
kde = gaussian_kde(xz_pts.T, bw_method="silverman")
density = kde(_grid_pts).reshape(_G, _G)
density[~_inside_mask] = 0.0
inside_vals = density[_inside_mask]
d_max = inside_vals.max()
if d_max > 0:
# Normalise by 95th-percentile density so a tight peak at one
# corner doesn't wash out the rest of the triangle. Values
# above the cap are clipped to the max alpha.
d_ref = np.percentile(inside_vals[inside_vals > 0], 95)
if d_ref == 0:
d_ref = d_max
alpha_grid = np.clip(density / d_ref, 0, 1) * 0.50
# Build one quad per inside cell, coloured by density alpha
quads, face_cols = [], []
for i in range(_G):
for j in range(_G):
if not _inside_mask[i, j]:
continue
xi, zj = _x_lin[j], _z_lin[i]
x0, x1 = xi - _dx / 2, xi + _dx / 2
z0, z1 = zj - _dz / 2, zj + _dz / 2
quads.append([(x0, y, z0), (x1, y, z0),
(x1, y, z1), (x0, y, z1)])
fc = rgba_base.copy()
fc[3] = float(alpha_grid[i, j])
face_cols.append(fc)
heat = Poly3DCollection(quads, facecolors=face_cols,
edgecolors="none", zorder=0)
ax.add_collection3d(heat)
# Iso-prob grid lines
for level in (0.25, 0.5, 0.75):
for vi in range(3):
v0 = _TRI3D_VERTS[vi]
v1 = _TRI3D_VERTS[(vi + 1) % 3]
v2 = _TRI3D_VERTS[(vi + 2) % 3]
p1 = level * v0 + (1 - level) * v1
p2 = level * v0 + (1 - level) * v2
ax.plot([p1[0], p2[0]], [y, y], [p1[1], p2[1]],
color="#cccccc", linewidth=0.4, linestyle="--",
alpha=0.45, zorder=1)
# ── vertex labels just outside the front face of the merged volume ────────
# Place labels at the H-layer plane (y=0, front-left) but pushed slightly
# in front of the y-axis so they don't collide with the wireframe.
lbl_info = [
(0, -0.13, -0.08), # H: bottom-left
(1, 0.00, 0.10), # G: top
(2, 0.13, -0.08), # S: bottom-right
]
front_y = min(y_pos.values()) - 0.08
for ci, dx, dz in lbl_info:
vx, vz = _TRI3D_VERTS[ci]
ax.text(vx + dx, front_y, vz + dz,
_CLASS_LABELS.get(ci, f"C{ci}"),
fontsize=10, fontweight="bold", ha="center", va="center",
zorder=10)
# ── data dots ─────────────────────────────────────────────────────────────
for fold_idx, df in enumerate(frames):
prob_cols = [f"prob_{head}_c{c}" for c in range(3)]
probs_all = df[prob_cols].values
y_true = df["y_true"].values.astype(int)
for true_cls in all_true:
mask = y_true == true_cls
if not mask.any():
continue
probs = probs_all[mask]
xz = _bary_to_cart_3d(probs) # Nx2 (x, z)
noise = rng.normal(0, 0.004, xz.shape)
y_vals = np.full(mask.sum(), y_pos[true_cls])
ax.scatter(xz[:, 0] + noise[:, 0],
y_vals,
xz[:, 1] + noise[:, 1],
color=colours[true_cls][fold_idx],
s=22, alpha=0.82, linewidths=0,
depthshade=False, zorder=5)
ax.set_xlim(-0.15, 1.15)
ax.set_ylim(-0.3, max(y_pos.values()) + 0.3)
ax.set_zlim(-0.1, np.sqrt(3) / 2 + 0.1)
ax.set_xlabel("")
ax.set_ylabel("")
ax.set_zlabel("")
ax.set_xticks([])
ax.set_yticks([])
ax.set_zticks([])
ax.xaxis.pane.fill = False
ax.yaxis.pane.fill = False
ax.zaxis.pane.fill = False
ax.grid(False)
# ── legend: same grid style as 2D plots, placed as an inset axes ─────────
ax_leg = fig.add_axes([0.68, 0.65, 0.28, 0.30])
_draw_grid_legend(ax_leg, n_folds, all_true, colours, title="True class")
fig.tight_layout()
return _save(fig, out_dir or run_dir / "plots", f"prob_triangle3d_{head}.png")
# ── shared save helper ─────────────────────────────────────────────────────────
def _save(fig: plt.Figure, out_dir: Path, filename: str) -> Path:
out_dir = Path(out_dir)
out_dir.mkdir(parents=True, exist_ok=True)
out_path = out_dir / filename
fig.savefig(out_path, dpi=150, bbox_inches="tight")
plt.close(fig)
print(f"Saved: {out_path}")
return out_path
# ── CLI ────────────────────────────────────────────────────────────────────────
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--run-dir", required=True)
ap.add_argument("--head", default=None,
choices=["fused", "fused_head", "img", "md"],
help="Head to plot. Omit to auto-detect and plot all available heads.")
ap.add_argument("--style", default="strips",
choices=["strips", "confidence", "triangle", "triangle3d"],
help="strips: X=true class, Y=P(Glaucoma). "
"confidence: X=predicted class × true sub-column, Y=confidence. "
"triangle: ternary simplex plot (multiclass only).")
ap.add_argument("--out", default=None)
args = ap.parse_args()
rd = Path(args.run_dir)
od = Path(args.out) if args.out else None
heads = [args.head] if args.head else _detect_heads(rd)
print(f"Heads to plot: {heads}")
plot_fn = {
"confidence": plot_confidence,
"triangle": plot_triangle,
"triangle3d": plot_triangle_3d,
}.get(args.style, plot_strip)
for head in heads:
print(f"\n--- {head} ---")
try:
plot_fn(rd, head=head, out_dir=od)
except Exception as exc:
print(f" [skip] {head}: {exc}")
if __name__ == "__main__":
main()
scripts/output_analysis/visualizations/plot_run_roc_v2.py
+38 -35
View File
@@ -57,12 +57,10 @@ _PROBS_PRIORITY: dict[str, list[str]] = {
_ALL_PROBS = ["probs_fused_head", "probs_fused", "probs_bilat", "probs_classic"]
def detect_probs_stem(fold_dir: Path, tower_mode: str | None) -> str | None:
def detect_probs_stems(fold_dir: Path, tower_mode: str | None) -> list[str]:
"""Return all present probs stems (in priority order) for this fold dir."""
priority = _PROBS_PRIORITY.get(tower_mode, _ALL_PROBS) if tower_mode else _ALL_PROBS
for stem in priority:
if (fold_dir / f"{stem}.npy").exists():
return stem
return None
return [stem for stem in priority if (fold_dir / f"{stem}.npy").exists()]
# ---------------------------------------------------------------------------
@@ -246,39 +244,44 @@ def main() -> None:
if not fold_dirs:
raise SystemExit(f"No fold subdirectories found in {mode_dir}")
# Determine probs stem
probs_stem = args.probs
if probs_stem is None:
# Determine probs stems to plot
if args.probs is not None:
stems_to_plot = [args.probs]
else:
# Collect all stems present across any fold dir
seen: list[str] = []
for fd in fold_dirs:
probs_stem = detect_probs_stem(fd, args.tower_mode)
if probs_stem:
break
if probs_stem is None:
raise SystemExit(f"Could not detect a probs file in {mode_dir}/fold*/")
print(f"Using probs: {probs_stem}.npy")
# Load all folds
per_fold: list[tuple[int, dict]] = []
for fd in fold_dirs:
fold_idx = int(fd.name.replace("fold", ""))
result = load_fold(fd, probs_stem, args.eval_mode)
if result is None:
print(f" [skip] fold {fold_idx}: missing y_true or {probs_stem}.npy")
continue
y, p = result
curves = per_class_roc(y, p)
per_fold.append((fold_idx, curves))
auc_str = " ".join(
f"class{k}={v['auc']:.3f}" for k, v in curves.items()
)
print(f" fold {fold_idx}: {auc_str}")
if not per_fold:
raise SystemExit("No usable folds — nothing to plot.")
for s in detect_probs_stems(fd, args.tower_mode):
if s not in seen:
seen.append(s)
stems_to_plot = seen
if not stems_to_plot:
raise SystemExit(f"Could not detect any probs file in {mode_dir}/fold*/")
print(f"Probs stems to plot: {stems_to_plot}")
out_dir = mode_dir / "plots"
plot_perfold(per_fold, out_dir, args.class_names, probs_stem, args.eval_mode)
plot_mean_ovr(per_fold, out_dir, args.class_names, probs_stem, args.eval_mode)
for probs_stem in stems_to_plot:
print(f"\n--- {probs_stem} ---")
per_fold: list[tuple[int, dict]] = []
for fd in fold_dirs:
fold_idx = int(fd.name.replace("fold", ""))
result = load_fold(fd, probs_stem, args.eval_mode)
if result is None:
print(f" [skip] fold {fold_idx}: missing y_true or {probs_stem}.npy")
continue
y, p = result
curves = per_class_roc(y, p)
per_fold.append((fold_idx, curves))
auc_str = " ".join(f"class{k}={v['auc']:.3f}" for k, v in curves.items())
print(f" fold {fold_idx}: {auc_str}")
if not per_fold:
print(f" No usable folds for {probs_stem}, skipping.")
continue
plot_perfold(per_fold, out_dir, args.class_names, probs_stem, args.eval_mode)
plot_mean_ovr(per_fold, out_dir, args.class_names, probs_stem, args.eval_mode)
print(f"\nPlots written to {out_dir}")
scripts/utility/backup_mirror_with_archive.sh
+35
View File
@@ -44,7 +44,42 @@ cmd=(
--delete
--backup
--backup-dir="$archive_dir"
# Directories with no backup value
--exclude=".git/"
--exclude=".claude/"
--exclude=".archive/"
# Large datasets stored elsewhere
--exclude="refuge/"
--exclude="Refuge/"
--exclude="REFUGE/"
--exclude="papila/"
--exclude="Papila/"
--exclude="PAPILA/"
# Python / general caches
--exclude="__pycache__/"
--exclude=".mypy_cache/"
--exclude=".ruff_cache/"
--exclude=".pytest_cache/"
--exclude=".cache/"
--exclude="*.pyc"
--exclude="*.pyo"
# Virtual environments
--exclude=".venv/"
--exclude="venv/"
--exclude="env/"
# Node
--exclude="node_modules/"
# Build / dist artifacts
--exclude="*.egg-info/"
--exclude="dist/"
--exclude="build/"
--exclude=".eggs/"
# IDE / editor metadata
--exclude=".idea/"
--exclude=".vscode/"
# OS metadata
--exclude=".DS_Store"
--exclude="Thumbs.db"
"${SOURCE%/}/"
"${DEST%/}/"
)
setup_env.sh
Executable
+64
View File
@@ -0,0 +1,64 @@
#!/usr/bin/env bash
# ============================================================
# HyperTower environment setup script.
#
# Installs all required packages for the v3 pipeline.
# PyTorch is installed CPU-only by default.
#
# To upgrade to GPU after running this script:
# NVIDIA (CUDA 12.8):
# pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128
# AMD (ROCm 6.2):
# pip install torch torchvision --index-url https://download.pytorch.org/whl/rocm6.2
#
# Usage:
# bash setup_env.sh
# ============================================================
set -e
echo "=== HyperTower environment setup ==="
echo "Installing CPU-only PyTorch (upgrade separately for GPU)"
echo ""
# ── PyTorch (CPU) ─────────────────────────────────────────────
pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
# ── Core data science ─────────────────────────────────────────
pip install \
numpy \
pandas \
scikit-learn \
scipy
# ── Image processing ──────────────────────────────────────────
pip install \
Pillow \
scikit-image
# ── Visualisation ─────────────────────────────────────────────
pip install \
matplotlib
# ── Distributed job system ────────────────────────────────────
pip install \
fastapi \
"uvicorn[standard]" \
requests \
pydantic
# ── Statistics ────────────────────────────────────────────────
pip install \
statsmodels
# ── Utilities ─────────────────────────────────────────────────
pip install \
tqdm \
openpyxl
echo ""
echo "=== Setup complete ==="
echo ""
echo "To enable GPU support:"
echo " NVIDIA: pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128"
echo " AMD: pip install torch torchvision --index-url https://download.pytorch.org/whl/rocm6.2"
v3/PLAN_v4_hypertower.md
+179
View File
@@ -0,0 +1,179 @@
# v4 HyperTower Planning Document
## Goals
Rebuild the orchestrator using `run_ntower_cv` as the architectural foundation, with four key improvements: JSON config, decoupled data sources, declarative tower lists, and a cross-tower communication protocol.
---
## 1. JSON Config (replace argparse)
The orchestrator receives a single JSON config file. It has no hardcoded knowledge of what args individual towers or data modules need — it just forwards the relevant subtrees.
```json
{
"run_name": "v4/ensemble_fused",
"eval_mode": "binary",
"epochs": 30,
"fusion_epochs": 10,
"fold_seed": 100,
"seed": 1234,
"data": {
"module": "v4.papila.v4papila",
"args": {
"iop_corr_method": "ratio",
"iop_drop_raw": true,
"exclude_cols": ["Axial_Length"]
}
},
"towers": [
{
"name": "img",
"module": "v3.classes.image_towers",
"class": "ImageEncoder",
"args": { "backbone": "refugelike", "freeze_ratio": 0.5, "augment": true },
"warmup_epochs": 0
},
{
"name": "cd",
"module": "v3.classes.clinical_towers",
"class": "ClinicalEncoder",
"args": { "hidden_dim": 128 },
"warmup_epochs": 40
}
],
"bridge": {
"mode": "embedding_mlp",
"fusion_dim": 256,
"hidden_dim": 256
},
"training": {
"lr": 1e-4,
"batch_size": 16,
"bcd_prob": 0.5,
"warmup_tower_epochs": 3,
"warmup_fused_epochs": 3
}
}
```
The orchestrator loads this with `json.load`, then calls `importlib.import_module(cfg["data"]["module"]).build_data(cfg["data"]["args"])` and similarly instantiates towers. No argparse anywhere in the orchestrator.
---
## 2. Decoupled Data Sources
`v3/classes/papila_builders.py` → clone to `v4/papila/v4papila.py`.
Merge in the relevant logic from `papila_data.py` (preprocessing, feature typing, IOP correction, etc.) so `v4papila.py` is self-contained.
Contract: every data module must expose:
```python
def build_data(args: dict) -> DataBundle:
...
```
The orchestrator calls `build_data` and gets back a `DataBundle`. It knows nothing else about the data source. Future modules (e.g. `v4/eyepacs/eyepacs_data.py`) just implement the same function.
---
## 3. Declarative Tower List
Towers are loaded from the `"towers"` list in the JSON and stored as an ordered dict keyed by `name`. The orchestrator never imports a tower class directly.
```python
towers = {}
for t_cfg in cfg["towers"]:
mod = importlib.import_module(t_cfg["module"])
cls = getattr(mod, t_cfg["class"])
# some tower constructors need data (e.g. ClinicalEncoder needs feature_dim)
# pass data as an optional kwarg; tower ignores it if not needed
towers[t_cfg["name"]] = cls(data=data, **t_cfg["args"])
```
Tower-specific training metadata (warmup epochs, batch key) lives entirely in the JSON, not in the orchestrator.
---
## 4. Cross-Tower Communication: `early_pass` Protocol
**Problem:** GeometryTower needs to precompute segmentation maps from images, then inject them into other towers' sample dicts before loaders are built. This is currently done imperatively in the orchestrator.
**Proposed solution: `early_pass` connector interface**
Each tower optionally implements:
```python
class TowerBase:
def early_pass(self, context: EarlyPassContext) -> None:
"""Called once per fold before loaders are built.
Tower can read from / write to shared context."""
pass
```
`EarlyPassContext` is a shared mutable object passed to all towers in order:
```python
@dataclass
class EarlyPassContext:
eye_train: list[dict]
bilat_train: list[dict]
bilat_val: list[dict]
bilat_test: list[dict]
image_preprocessor: object
image_cache: object
device: torch.device
store: dict = field(default_factory=dict) # cross-tower key-value store
```
Example: GeometryTower's `early_pass` computes seg maps and injects them into the sample dicts directly (modifying `eye_train` etc. in place), exactly as it does today — but now the orchestrator just calls:
```python
for tower in towers.values():
tower.early_pass(context)
```
The cross-talk case the user described (img_tower outputs geometry → cd_tower reads it) uses `context.store`:
```python
# ImageTower.early_pass:
context.store["geometry_maps"] = self._compute_geometry(context)
# ClinicalTower.early_pass:
geo = context.store.get("geometry_maps")
if geo is not None:
self._inject_geometry(context, geo)
```
Tower ordering in the JSON list determines execution order, so dependencies are declared implicitly. If a tower has no `early_pass`, the default no-op in `TowerBase` is used.
**Alternative considered:** explicit dependency graph / DAG execution. Rejected for now — JSON ordering is simpler and sufficient for current needs. Can revisit if cross-tower dependencies become non-linear.
---
## 5. File Layout
```
v4/
hypertower/
v4_hypertower.py # orchestrator (no argparse, no tower imports)
split_manager.py # copy/adapt from v3 (or just import)
papila/
v4papila.py # merged papila_builders + papila_data
configs/
ensemble_fused.json # example config
```
Existing `v3/classes/` tower implementations are reused directly — no duplication needed since they're importable by the JSON `"module"` field.
---
## 6. Open Questions / Decisions Needed
- **DataBundle API**: Does `build_data` need to return anything beyond the current `DataBundle`? Or should `DataBundle` grow a `profile` factory method?
- **Per-tower batch_key convention**: Currently `EYE_KEY_MAP = {"img": "image_1", "cd": "matrix_1"}` is hardcoded. Should this be declared in the tower JSON config or inferred from tower type?
- **cd_warmup loader**: Slot stripping (`if k != "image_1"`) is currently img-tower-aware. Under the new design, each tower should declare which slots it needs for warmup vs full training, so the orchestrator can build the right loader without knowing about `image_1`.
- **Geometry injection today vs `early_pass`**: Geometry currently mutates sample dicts; `early_pass` formalizes this. Needs a migration plan for existing GeometryTower.
---
## Implementation Order (once ntower_cv is validated)
1. Write `v4papila.py` (merge papila_builders + papila_data, expose `build_data(args)`)
2. Add `early_pass(context)` no-op to `TowerBase`; implement in `GeometryTower`
3. Write `v4_hypertower.py` orchestrator using JSON config + importlib tower loading
4. Port one config (ensemble_fused) end-to-end and compare outputs against ntower_cv
v3/__init__.py
View File
v3/classes/SE_attention.py
Executable
+123
View File
@@ -0,0 +1,123 @@
# se_block.py
import torch
import torch.nn as nn
class SEGateLogger:
"""
Lightweight stats over SE gates.
Use: logger.accumulate(gates) each batch; logger.get() at epoch end.
"""
def __init__(self, enabled: bool = True, track_channels: bool = False, dim: int | None = None):
self.enabled = enabled
self.track_channels = track_channels
self.dim = dim
self.reset()
def reset(self):
self._n = 0
self._sum = 0.0
self._sum2 = 0.0
self._lt02 = 0
self._gt08 = 0
# optional per-channel
self._ch_sum = None
self._ch_count = 0
if self.track_channels and self.dim is not None:
self._ch_sum = torch.zeros(self.dim, dtype=torch.float32)
@torch.no_grad()
def accumulate(self, gates: torch.Tensor):
if not self.enabled:
return
# gates expected shape [N, C]; if a map/sequence gate is passed, reduce to [N, C]
if gates.dim() == 4: # [N,C,H,W] gates (uncommon)
g = gates.mean(dim=(2,3))
elif gates.dim() == 3: # [N,T,C] gates (sequence)
g = gates.mean(dim=1)
elif gates.dim() == 2: # [N,C]
g = gates
else:
g = gates.view(gates.size(0), -1)
g = g.detach()
self._n += g.numel()
self._sum += g.sum().item()
self._sum2 += (g*g).sum().item()
self._lt02 += (g < 0.2).sum().item()
self._gt08 += (g > 0.8).sum().item()
if self._ch_sum is not None:
self._ch_sum += g.sum(dim=0).cpu()
self._ch_count += g.size(0)
def get(self, reset: bool = True):
if self._n == 0:
return None
mean = self._sum / self._n
var = max(0.0, self._sum2 / self._n - mean * mean)
out = {
"mean": mean,
"std": var ** 0.5,
"pct_lt_0.2": self._lt02 / self._n,
"pct_gt_0.8": self._gt08 / self._n,
}
if self._ch_sum is not None and self._ch_count > 0:
out["channel_mean"] = (self._ch_sum / float(self._ch_count)).tolist()
if reset:
self.reset()
return out
class SEBlock(nn.Module):
"""
SE-style channel gating that works for vectors and maps.
Input:
- [N, C] (vector) -> squeeze = identity
- [N, C, H, W] (image map) -> squeeze over H,W
- [N, T, C] (sequence) -> squeeze over T
Gate modes:
- residual (default): gate = 1 + tanh(MLP(s)) in (0, 2) [identity at init]
- plain: gate = sigmoid(MLP(s)) in (0, 1)
"""
def __init__(self, dim: int, reduction: int = 16, residual: bool = True, identity_init: bool = True):
super().__init__()
hid = max(1, dim // max(1, reduction))
self.fc1 = nn.Linear(dim, hid, bias=True)
self.act = nn.ReLU(inplace=True)
self.fc2 = nn.Linear(hid, dim, bias=True)
self.residual = residual
if residual and identity_init:
# make MLP output ~0 at start → gate ≈ 1.0
nn.init.zeros_(self.fc2.weight)
nn.init.zeros_(self.fc2.bias)
def _squeeze(self, x: torch.Tensor) -> torch.Tensor:
if x.dim() == 2: # [N,C]
return x
if x.dim() == 4: # [N,C,H,W]
return x.mean(dim=(2,3))
if x.dim() == 3: # [N,T,C]
return x.mean(dim=1)
# fallback: flatten non-batch dims into channels
return x.view(x.size(0), -1)
def _broadcast(self, gate: torch.Tensor, like: torch.Tensor) -> torch.Tensor:
if like.dim() == 2:
return gate
if like.dim() == 3:
return gate.unsqueeze(1) # [N,1,C]
if like.dim() == 4:
return gate.unsqueeze(-1).unsqueeze(-1) # [N,C,1,1]
return gate.view_as(like)
def forward(self, x: torch.Tensor):
s = self._squeeze(x) # [N,C]
u = self.fc2(self.act(self.fc1(s))) # [N,C]
if self.residual:
gate = 1.0 + torch.tanh(u) # (0, 2) with identity at 1.0
else:
gate = torch.sigmoid(u) # (0, 1)
y = x * self._broadcast(gate, x)
return y, gate # return both the reweighted tensor and the gate for logging
v3/classes/__init__.py
+137
View File
@@ -0,0 +1,137 @@
from .network_manager import (
FoldResult,
LoaderBundle,
NetworkManager,
PatientSplit,
)
from .split_manager import (
PatientFirstSplitManager,
SplitPlan,
build_patient_split_plans,
)
from .profiles import (
DatasetProfile,
SimpleDatasetProfile,
SlotDescriptor,
PapilaProfile,
build_papila_profile,
)
from .loader_factory import SlotLoaderFactory
from .slot_dataset import SlotDataset, slot_collate
from .papila_data import PapilaData
from .papila_builders import build_papila_data
from .data_bundle import DataBundle
from .transforms import (
ImageTransformConfig,
backbone_transform_config,
build_backbone_transform,
build_eval_transform,
build_imagenet_transform,
ResizeTransform,
CenterCropTransform,
ROICropTransform,
JitterBundleTransform,
UnetMaskProvider,
TRANSFORM_REGISTRY,
build_transform_chain,
)
from .towerbase import TowerBase, build_backbone, train_towers_epoch, collect_probs_towers
from .image_towers import ImageEncoder, SiameseImageTower, ImageTower
from .clinical_towers import ClinicalEncoder, ClinicalDataTower
from .geometry_towers import GeometryTower
from .hypertower_models import (
SingleEyeHT,
BilateralHT,
SiameseHT,
FusedEnsembleHT,
LogitMLPEnsembleHT,
EmbeddingMLPEnsembleHT,
NTowerHT,
NLateralHT,
MonoTowerHT,
train_single_epoch,
train_bilateral_epoch,
train_siamese_epoch,
train_fusion_epoch,
train_ntower_epoch,
train_mono_epoch,
collect_probs_classic,
collect_probs_ensemble,
collect_probs_bilateral,
collect_probs_siamese,
collect_probs_ntower,
collect_probs_mono,
V2ModeComparisonOps,
)
from .bridges import Bridge, HTClassifier, HyperBridge, VoteBridge
from .hypertower_logger import HypertowerLogger
__all__ = [
"NetworkManager",
"PatientSplit",
"LoaderBundle",
"FoldResult",
"PatientFirstSplitManager",
"SplitPlan",
"build_patient_split_plans",
"DatasetProfile",
"SimpleDatasetProfile",
"SlotDescriptor",
"PapilaProfile",
"build_papila_profile",
"PapilaData",
"build_papila_data",
"DataBundle",
"SlotLoaderFactory",
"SlotDataset",
"slot_collate",
"ImageTransformConfig",
"backbone_transform_config",
"build_backbone_transform",
"build_eval_transform",
"build_imagenet_transform",
"ResizeTransform",
"CenterCropTransform",
"ROICropTransform",
"JitterBundleTransform",
"UnetMaskProvider",
"TRANSFORM_REGISTRY",
"build_transform_chain",
"TowerBase",
"build_backbone",
"train_towers_epoch",
"collect_probs_towers",
"ImageEncoder",
"SiameseImageTower",
"ImageTower",
"ClinicalEncoder",
"ClinicalDataTower",
"GeometryTower",
"SingleEyeHT",
"BilateralHT",
"SiameseHT",
"FusedEnsembleHT",
"LogitMLPEnsembleHT",
"EmbeddingMLPEnsembleHT",
"NTowerHT",
"NLateralHT",
"MonoTowerHT",
"train_single_epoch",
"train_bilateral_epoch",
"train_siamese_epoch",
"train_fusion_epoch",
"train_ntower_epoch",
"train_mono_epoch",
"collect_probs_classic",
"collect_probs_ensemble",
"collect_probs_bilateral",
"collect_probs_siamese",
"collect_probs_ntower",
"collect_probs_mono",
"Bridge",
"HTClassifier",
"HyperBridge",
"VoteBridge",
"V2ModeComparisonOps",
"HypertowerLogger",
]
v3/classes/backbones.py
Executable
+178
View File
@@ -0,0 +1,178 @@
# classes/backbones.py
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Callable, Dict, List
import torch
from torch import nn
from torchvision import models
@dataclass(frozen=True)
class BackboneSpec:
ctor: Callable # torchvision constructor
weights_default: object # torchvision Weights enum DEFAULT member
strip: Callable[[nn.Module], tuple] # fn(model)->(out_dim, model_no_head)
blocks: Callable[[nn.Module], List[nn.Module]] # fn(model)->ordered blocks for freezing
REFUGELIKE_BACKBONE_PATH = Path("models/v2/refuge/refugelike_backbone.pt")
REFUGE_DENSENET_PATH = Path("models/refuge/classifier/refuge_densenet_backbone.pt")
REFUGE_EFFICIENT_B0_PATH = Path("models/refuge/classifier/refuge_efficient_b0_backbone.pt")
REFUGE_EFFICIENT_B7_PATH = Path("models/refuge/classifier/refuge_efficient_b7_backbone.pt")
# --- strip fns ---
def _strip_efficientnet_b0(m: models.EfficientNet):
from torch import nn as _nn
out_dim = m.classifier[1].in_features
m.classifier = _nn.Identity()
return out_dim, m
def _strip_resnet(m: models.ResNet):
out_dim = m.fc.in_features
m.fc = nn.Identity()
return out_dim, m
def _strip_densenet(m: models.DenseNet):
out_dim = m.classifier.in_features
m.classifier = nn.Identity()
return out_dim, m
def _strip_vgg(m: models.VGG):
out_dim = m.classifier[0].in_features # 25088 for VGG16 at 224×224
m.classifier = nn.Identity()
return out_dim, m
def _strip_mobilenet_v2(m: models.MobileNetV2):
out_dim = m.classifier[1].in_features
m.classifier = nn.Identity()
return out_dim, m
def _strip_inception_v3(m: models.Inception3):
out_dim = m.fc.in_features
m.fc = nn.Identity()
m.aux_logits = False
m.AuxLogits = None # torchvision checks `AuxLogits is not None`, not the flag
return out_dim, m
# --- block splitters for ratio-based freezing ---
def _blocks_efficientnet_b0(m: models.EfficientNet):
return list(m.features)
def _blocks_resnet(m: models.ResNet):
stem = nn.Sequential(m.conv1, m.bn1, m.relu, m.maxpool)
return [stem, m.layer1, m.layer2, m.layer3, m.layer4]
def _blocks_densenet(m: models.DenseNet):
f = m.features
stem = nn.Sequential(f.conv0, f.norm0, f.relu0, f.pool0)
return [stem, f.denseblock1, f.transition1, f.denseblock2, f.transition2,
f.denseblock3, f.transition3, f.denseblock4, f.norm5]
def _blocks_vgg(m: models.VGG):
stages, cur = [], []
for mod in m.features:
cur.append(mod)
if isinstance(mod, nn.MaxPool2d):
stages.append(nn.Sequential(*cur)); cur = []
if cur: stages.append(nn.Sequential(*cur))
return stages
def _blocks_mobilenet_v2(m: models.MobileNetV2):
return list(m.features)
def _blocks_inception_v3(m: models.Inception3):
blocks = []
for name, child in m.named_children():
if name in ("fc", "AuxLogits"):
continue
blocks.append(child)
return blocks
# --- registry (covers paper models available in torchvision) ---
BACKBONES: Dict[str, BackboneSpec] = {
"efficientnet_b0": BackboneSpec(
ctor=models.efficientnet_b0,
weights_default=models.EfficientNet_B0_Weights.DEFAULT,
strip=_strip_efficientnet_b0,
blocks=_blocks_efficientnet_b0,
),
"resnet50": BackboneSpec(
ctor=models.resnet50,
weights_default=models.ResNet50_Weights.DEFAULT,
strip=_strip_resnet,
blocks=_blocks_resnet,
),
"densenet121": BackboneSpec(
ctor=models.densenet121,
weights_default=models.DenseNet121_Weights.DEFAULT,
strip=_strip_densenet,
blocks=_blocks_densenet,
),
"vgg16": BackboneSpec(
ctor=models.vgg16,
weights_default=models.VGG16_Weights.DEFAULT,
strip=_strip_vgg,
blocks=_blocks_vgg,
),
"mobilenet_v2": BackboneSpec(
ctor=models.mobilenet_v2,
weights_default=models.MobileNet_V2_Weights.DEFAULT,
strip=_strip_mobilenet_v2,
blocks=_blocks_mobilenet_v2,
),
"inception_v3": BackboneSpec(
ctor=models.inception_v3,
weights_default=models.Inception_V3_Weights.DEFAULT,
strip=_strip_inception_v3,
blocks=_blocks_inception_v3,
),
"refugelike": BackboneSpec(
ctor=models.resnet50,
weights_default=None,
strip=_strip_resnet,
blocks=_blocks_resnet,
),
"refuge_densenet": BackboneSpec(
ctor=models.densenet121,
weights_default=None,
strip=_strip_densenet,
blocks=_blocks_densenet,
),
"refuge_efficient_b0": BackboneSpec(
ctor=models.efficientnet_b0,
weights_default=None,
strip=_strip_efficientnet_b0,
blocks=_blocks_efficientnet_b0,
),
"refuge_efficient_b7": BackboneSpec(
ctor=models.efficientnet_b7,
weights_default=None,
strip=_strip_efficientnet_b0,
blocks=_blocks_efficientnet_b0,
),
# Xception isn’t in torchvision
}
def list_names() -> List[str]:
return list(BACKBONES.keys())
def load_backbone_weights(key: str, model: nn.Module) -> None:
if key == "refugelike":
path = REFUGELIKE_BACKBONE_PATH
elif key == "refuge_densenet":
path = REFUGE_DENSENET_PATH
elif key == "refuge_efficient_b0":
path = REFUGE_EFFICIENT_B0_PATH
elif key == "refuge_efficient_b7":
path = REFUGE_EFFICIENT_B7_PATH
else:
return
if not path.exists():
raise FileNotFoundError(
"Custom REFUGE backbone not found at "
f"{path}. Export it via refuge_build.py --export-backbone first."
)
state = torch.load(path, map_location="cpu")
model.load_state_dict(state, strict=False)
v3/classes/bridges.py
+306
View File
@@ -0,0 +1,306 @@
from __future__ import annotations
import torch
import torch.nn as nn
from v3.classes.SE_attention import SEBlock, SEGateLogger
# ---------------------------------------------------------------------------
# HTClassifier — standalone classification head
# ---------------------------------------------------------------------------
class HTClassifier(nn.Module):
"""Minimal classification head: ReLU → Dropout → Linear(in_dim → num_classes).
Used as the output stage of Bridge, HyperBridge, and any vehicle that needs
a reusable, identifiable classifier type.
"""
def __init__(self, in_dim: int, num_classes: int, dropout: float = 0.5):
super().__init__()
self.head = nn.Sequential(
nn.ReLU(),
nn.Dropout(dropout),
nn.Linear(in_dim, num_classes),
)
def forward(self, z: torch.Tensor) -> torch.Tensor:
return self.head(z)
# ---------------------------------------------------------------------------
# Bridge — N-tower fusion
# ---------------------------------------------------------------------------
class Bridge(nn.Module):
"""
N-tower fusion bridge.
Takes a list of tower embeddings, projects each to a common ``fusion_dim``,
element-wise multiplies all projections, optionally applies an SE gate, then
classifies the fused representation via an HTClassifier.
Each tower also gets an auxiliary classification head (used for BCD training).
Construction
------------
``tower_dims`` is an ordered list of embedding dimensionalities — one entry per
embedding slot that will be passed to ``fuse()`` or ``forward()``.
Tower slots are accessed by index: ``W[i]``, ``ln[i]``, ``aux_heads[i]``.
The bridge has no knowledge of what modality each slot carries.
"""
def __init__(
self,
tower_dims: list[int],
num_classes: int,
fusion_dim: int = 256,
mode: str = "fused",
dropout: float = 0.5,
use_se: bool = True,
se_reduction: int = 16,
se_pre_norm: bool = True,
):
super().__init__()
self.mode = mode
self.use_se = use_se
self.tower_dims = list(tower_dims)
# Per-tower projection heads: each projects dim_i → fusion_dim
self.W = nn.ModuleList([nn.Linear(d, fusion_dim) for d in tower_dims])
self.ln = nn.ModuleList(
[nn.LayerNorm(fusion_dim) if se_pre_norm else nn.Identity()
for _ in tower_dims]
)
# Per-tower auxiliary classifiers (for BCD training)
self.aux_heads = nn.ModuleList([nn.Linear(d, num_classes) for d in tower_dims])
# SE gate on the fused vector
self.se = SEBlock(fusion_dim, reduction=se_reduction, residual=True) if use_se else None
self.se_log = SEGateLogger(enabled=use_se, track_channels=False, dim=fusion_dim)
# Fused classifier head
self.classifier_fused = HTClassifier(fusion_dim, num_classes, dropout)
# ------------------------------------------------------------------
# SE helpers
# ------------------------------------------------------------------
def reset_se_stats(self) -> None:
"""Call at epoch start."""
if getattr(self, "se_log", None):
self.se_log.reset()
def get_se_stats(self, reset: bool = True):
"""Call after eval. Returns dict or None."""
if getattr(self, "se_log", None) and self.se_log.enabled:
return self.se_log.get(reset=reset)
return None
# ------------------------------------------------------------------
# Core fusion
# ------------------------------------------------------------------
def _compute_fused(self, embeddings: list[torch.Tensor]) -> torch.Tensor:
"""Return z_fused embedding (before classifier_fused)."""
assert len(embeddings) == len(self.W), (
f"Bridge expects {len(self.W)} embeddings, got {len(embeddings)}"
)
h = self.ln[0](self.W[0](embeddings[0]))
for i in range(1, len(embeddings)):
h = h * self.ln[i](self.W[i](embeddings[i]))
if self.se is not None:
h, gates = self.se(h)
if self.se_log.enabled:
self.se_log.accumulate(gates)
return h
# ------------------------------------------------------------------
# N-tower API
# ------------------------------------------------------------------
def fuse(
self, embeddings: list[torch.Tensor]
) -> tuple[torch.Tensor, list[torch.Tensor]]:
"""
N-tower forward pass.
Parameters
----------
embeddings : list of Tensor — one per tower slot (same order as tower_dims).
Returns
-------
logits_fused : Tensor [B, num_classes]
aux_logits : list of Tensor — one per tower slot, each [B, num_classes]
"""
z_fused = self._compute_fused(embeddings)
logits_fused = self.classifier_fused(z_fused)
aux = [head(e) for head, e in zip(self.aux_heads, embeddings)]
return logits_fused, aux
def encode(self, embeddings: list[torch.Tensor]) -> torch.Tensor:
"""Return z_fused without applying the classifier head."""
return self._compute_fused(embeddings)
def set_phase(self, phase: str) -> None:
"""
Set requires_grad on bridge sub-modules according to training phase.
- ``cd_warmup`` — freeze everything in the bridge
- ``tower_warmup`` — aux_heads trainable, projections + fused head frozen
- ``fused_warmup`` — projections + fused head trainable, aux_heads frozen
- ``main`` / other — everything trainable
"""
def _rg(module, enabled):
for p in module.parameters():
p.requires_grad = enabled
if phase == "cd_warmup":
_rg(self, False)
return
if phase == "tower_warmup":
for head in self.aux_heads:
_rg(head, True)
for W_i in self.W:
_rg(W_i, False)
for ln_i in self.ln:
_rg(ln_i, False)
_rg(self.classifier_fused, False)
if self.se is not None:
_rg(self.se, False)
return
if phase == "fused_warmup":
for head in self.aux_heads:
_rg(head, False)
for W_i in self.W:
_rg(W_i, True)
for ln_i in self.ln:
_rg(ln_i, True)
_rg(self.classifier_fused, True)
if self.se is not None:
_rg(self.se, True)
return
_rg(self, True)
def forward(
self, embeddings: list[torch.Tensor]
) -> tuple[torch.Tensor, list[torch.Tensor]]:
"""N-tower forward. Delegates to fuse()."""
return self.fuse(embeddings)
# ---------------------------------------------------------------------------
# HyperBridge — higher-order bridge over HT module outputs
# ---------------------------------------------------------------------------
class HyperBridge(nn.Module):
"""Higher-order bridge that fuses z_fused embeddings from multiple HT modules.
Operates at the HT output level (z_fused from each HT's Bridge.encode())
rather than raw tower embedding level.
Modes
-----
embedding_mlp (default)
Concatenate all z_fused inputs → MLP → logits.
Analogous to EmbeddingMLPEnsembleHT, generalised to N inputs.
``Linear(N*fusion_dim → hidden_dim) → ReLU → Dropout → Linear(hidden_dim → num_classes)``
classic_bridge
Project each input to ``hidden_dim``, Hadamard product, HTClassifier.
Analogous to Bridge operating at the HT level — handles inputs of
differing dims via per-input projection layers.
``W[i](z_i) → LayerNorm → Hadamard → ReLU → Dropout → Linear(hidden_dim → num_classes)``
Both modes expose per-input auxiliary HTClassifier heads for BCD-style training.
Parameters
----------
input_dims : ordered dict {name: dim} for each HT input.
In embedding_mlp mode, dims may differ.
In classic_bridge mode, all dims must be equal (shared space).
num_classes : output classes
hidden_dim : hidden dim for the embedding_mlp MLP head
mode : "embedding_mlp" | "classic_bridge"
dropout : dropout throughout
"""
def __init__(
self,
input_dims: dict[str, int],
num_classes: int,
hidden_dim: int = 256,
mode: str = "embedding_mlp",
dropout: float = 0.3,
):
super().__init__()
self.input_names = list(input_dims.keys())
self.mode = mode
dims = list(input_dims.values())
if mode == "embedding_mlp":
total_dim = sum(dims)
self.head = nn.Sequential(
nn.Linear(total_dim, hidden_dim), nn.ReLU(),
nn.Dropout(dropout), nn.Linear(hidden_dim, num_classes),
)
elif mode == "classic_bridge":
# Project each input to shared fusion_dim space, then Hadamard
self.W = nn.ModuleList([nn.Linear(d, hidden_dim) for d in dims])
self.ln = nn.ModuleList([nn.LayerNorm(hidden_dim) for _ in dims])
self.head = HTClassifier(hidden_dim, num_classes, dropout)
else:
raise ValueError(f"Unknown HyperBridge mode: {mode!r}")
# Per-input auxiliary classifiers (both modes)
self.aux_heads = nn.ModuleList([
HTClassifier(d, num_classes, dropout) for d in dims
])
def forward(
self,
inputs: dict[str, torch.Tensor],
) -> tuple[torch.Tensor, dict[str, torch.Tensor]]:
"""Fuse HT-level embeddings.
Parameters
----------
inputs : {name: z_fused [B, dim]} — z_fused from each HT's encode()
Returns
-------
logits : [B, num_classes]
aux_dict : {name: [B, num_classes]} — per-input aux head logits
"""
ordered = [inputs[name] for name in self.input_names]
if self.mode == "embedding_mlp":
logits = self.head(torch.cat(ordered, dim=1))
else: # classic_bridge: project → Hadamard → classify
h = self.ln[0](self.W[0](ordered[0]))
for i in range(1, len(ordered)):
h = h * self.ln[i](self.W[i](ordered[i]))
logits = self.head(h)
aux = {name: head(z) for name, head, z
in zip(self.input_names, self.aux_heads, ordered)}
return logits, aux
# ---------------------------------------------------------------------------
# VoteBridge
# ---------------------------------------------------------------------------
class VoteBridge(nn.Module):
def __init__(self, num_classes):
super().__init__()
self.vote_combiner = nn.Linear(num_classes * 2, num_classes)
def forward(self, out_img, out_md):
votes = torch.cat([out_img, out_md], dim=1)
return self.vote_combiner(votes)
v3/classes/clinical_towers.py
+162
View File
@@ -0,0 +1,162 @@
"""clinical_towers — ClinicalEncoder and ClinicalDataTower.
Self-contained: defines ClinicalEncoder directly (does not import it from
towers.py). Imports only TowerBase from towerbase plus infrastructure
(SEBlock, DataBundle).
"""
from __future__ import annotations
from typing import Optional
import torch
from torch import nn
from v3.classes.towerbase import TowerBase
from v3.classes.SE_attention import SEBlock
from v3.classes.data_bundle import DataBundle
# ---------------------------------------------------------------------------
# ClinicalEncoder — MLP over tabular features
# ---------------------------------------------------------------------------
class ClinicalEncoder(nn.Module):
"""MLP over DataBundle.vectorize_row outputs (converts to torch inside tower)."""
def __init__(
self,
clinical_data: DataBundle,
hidden_dim: int = 128,
dropout: float = 0.1,
use_se: bool = False,
se_reduction: int = 16,
se_pre_norm: bool = True,
):
super().__init__()
self.feature_dim = clinical_data.feature_dim
self.out_dim = hidden_dim
# Two-block MLP so we can optionally freeze/thaw per block.
self.block0 = nn.Sequential(
nn.Linear(self.feature_dim, hidden_dim),
nn.LayerNorm(hidden_dim),
nn.ReLU(inplace=True),
nn.Dropout(dropout),
)
self.block1 = nn.Sequential(
nn.Linear(hidden_dim, hidden_dim),
nn.ReLU(inplace=True),
)
self.net = nn.Sequential(self.block0, self.block1)
self.tower_ln = nn.LayerNorm(hidden_dim) if se_pre_norm else nn.Identity()
self.tower_se = (
SEBlock(hidden_dim, reduction=se_reduction, residual=True)
if use_se
else None
)
def forward(self, meta_np_or_torch) -> torch.Tensor:
if isinstance(meta_np_or_torch, torch.Tensor):
x = meta_np_or_torch
else:
x = torch.as_tensor(meta_np_or_torch, dtype=torch.float32)
h = self.net(x)
if self.tower_se is not None:
h, _ = self.tower_se(self.tower_ln(h))
return h
def set_freeze_ratio(self, ratio: float):
"""Optionally freeze earliest blocks of the MLP."""
r = max(0.0, min(1.0, float(ratio)))
for p in self.block0.parameters():
p.requires_grad = True
for p in self.block1.parameters():
p.requires_grad = True
if r >= 0.5:
for p in self.block0.parameters():
p.requires_grad = False
if r >= 1.0:
for p in self.block1.parameters():
p.requires_grad = False
# ---------------------------------------------------------------------------
# ClinicalDataTower — TowerBase implementation
# ---------------------------------------------------------------------------
class ClinicalDataTower(TowerBase, nn.Module):
"""
TowerBase implementation for the clinical metadata modality.
Wraps ClinicalEncoder (MLP over tabular features).
Contributes one embedding per eye slot: [z_cd].
Implements ``cd_warmup_embedding`` so train_towers_epoch can identify
this tower for cd_warmup phase via duck typing rather than isinstance checks.
"""
def __init__(
self,
*,
clinical_data,
cd_hidden_dim: int = 128,
cd_dropout: float = 0.1,
use_se: bool = False,
):
nn.Module.__init__(self)
self._encoder = ClinicalEncoder(
clinical_data=clinical_data,
hidden_dim=cd_hidden_dim,
dropout=cd_dropout,
use_se=use_se,
)
@property
def out_dim(self) -> int:
return self._encoder.out_dim
@property
def embed_dims(self) -> list[int]:
return [self._encoder.out_dim]
def set_phase(self, phase: str) -> None:
enabled = phase not in ("fused_warmup",)
for p in self._encoder.parameters():
p.requires_grad = enabled
def cd_warmup_embedding(
self,
batch: dict,
*,
device: torch.device,
) -> Optional[torch.Tensor]:
"""Return clinical embedding for slot 1, or None if matrix_1 is absent."""
m = batch.get("matrix_1")
if not torch.is_tensor(m):
return None
return self._encoder(m.to(device))
def embed_batch(
self,
batch: dict,
*,
device: torch.device,
slot: int = 1,
) -> list[torch.Tensor]:
m = batch.get(f"matrix_{slot}")
if not torch.is_tensor(m):
raise ValueError(f"ClinicalDataTower.embed_batch: matrix_{slot} is missing or not a tensor")
return [self._encoder(m.to(device))]
def prepare_fold(
self,
*,
eye_train,
bilat_train,
bilat_val,
bilat_test,
image_preprocessor,
image_cache,
device,
args,
) -> None:
pass # shares loader with ImageTower; no per-fold setup needed
v3/classes/croppers.py
+472
View File
@@ -0,0 +1,472 @@
"""Optic-disc image croppers and preprocessor factory for V2."""
from __future__ import annotations
from pathlib import Path
from typing import Dict, Optional, Tuple
import numpy as np
import pandas as pd
import torch
from PIL import Image, ImageDraw
from torchvision import transforms
from v3.classes.geometry_features import compute_geometry_features, disc_cup_from_mask_image
from v3.classes.unet_segmenter import UNetSegmenter
def _geometry_from_mask(mask: np.ndarray, scale: float) -> Dict:
mask = np.asarray(mask) > 0
coords = np.argwhere(mask)
if coords.size == 0:
raise RuntimeError("Empty mask; cannot derive geometry")
ys, xs = coords[:, 0], coords[:, 1]
centre_x = float(xs.mean())
centre_y = float(ys.mean())
width = float(xs.max() - xs.min())
height = float(ys.max() - ys.min())
diameter = max(width, height)
radius = diameter / 2.0
crop_radius = radius * scale
return {
"centre_x": centre_x,
"centre_y": centre_y,
"radius": radius,
"crop_radius": crop_radius,
"crop_size": crop_radius * 2.0,
}
class UNetImageCropper:
def __init__(
self,
manifest_path: Path,
weights_path: Path,
normalize: str = "per_image",
threshold: float = 0.5,
tta: bool = False,
scale: float = 2.5,
target_size: int = 224,
cache_dir: Optional[Path] = None,
) -> None:
self.segmenter = UNetSegmenter(
manifest_path=manifest_path,
normalize=normalize,
)
state = torch.load(weights_path, map_location=self.segmenter.device)
state_dict = state.get("model", state)
self.segmenter.model.load_state_dict(state_dict)
self.segmenter.model.to(self.segmenter.device)
self.segmenter.model.eval()
self.threshold = threshold
self.tta = tta
self.scale = scale
self.target_size = target_size
self.cache_dir = Path(cache_dir) if cache_dir is not None else None
if self.cache_dir is not None:
self.cache_dir.mkdir(parents=True, exist_ok=True)
self.to_tensor = transforms.ToTensor()
def _cache_path(self, image_path: Path) -> Optional[Path]:
if self.cache_dir is None:
return None
stem = image_path.stem
return self.cache_dir / f"{stem}_s{int(self.scale * 100)}.npz"
def clear_cache(self) -> None:
if self.cache_dir is None or not self.cache_dir.exists():
return
removed = sum(1 for f in self.cache_dir.glob("*.npz") if f.unlink() or True)
print(f"[UNetImageCropper] Cleared {removed} cached crop files from {self.cache_dir}")
def _infer_masks(self, image: Image.Image) -> Optional[Tuple[np.ndarray, np.ndarray]]:
resized = self.segmenter.preprocess_image(image)
tensor = self.segmenter._normalize_tensor(
self.to_tensor(resized).to(self.segmenter.device)
).unsqueeze(0)
with torch.no_grad():
logits = self.segmenter.model(tensor)
if self.tta:
t_h = torch.flip(tensor, dims=[3])
log_h = self.segmenter.model(t_h)
log_h = torch.flip(log_h, dims=[3])
t_v = torch.flip(tensor, dims=[2])
log_v = self.segmenter.model(t_v)
log_v = torch.flip(log_v, dims=[2])
logits = (logits + log_h + log_v) / 3.0
probs = torch.sigmoid(logits)[0].cpu().numpy()
disc_pred = (probs[0] > self.threshold).astype(np.uint8) * 255
cup_pred = (probs[1] > self.threshold).astype(np.uint8) * 255
disc_img = Image.fromarray(disc_pred, mode="L").resize(image.size, Image.NEAREST)
disc_mask = np.array(disc_img, dtype=np.uint8)
cup_img = Image.fromarray(cup_pred, mode="L").resize(image.size, Image.NEAREST)
cup_mask = (np.array(cup_img, dtype=np.uint8) > 0).astype(np.uint8)
cup_mask = (cup_mask > 0) & (disc_mask > 0)
cup_mask = cup_mask.astype(np.uint8)
disc_mask = (disc_mask > 0).astype(np.uint8)
return disc_mask, cup_mask
def _compute_crop_info(self, image: Image.Image, image_path: Path) -> Optional[dict]:
image_path = Path(image_path).resolve()
cache_path = self._cache_path(image_path)
cached_bounds = None
if cache_path is not None and cache_path.exists():
data = np.load(cache_path, allow_pickle=False)
try:
cached_bounds = {
"left": float(data["left"]),
"upper": float(data["upper"]),
"right": float(data["right"]),
"lower": float(data["lower"]),
}
if "features" in data.files:
cached_bounds["features"] = data["features"].astype(np.float32)
return cached_bounds
except KeyError:
cached_bounds = None
masks = self._infer_masks(image)
if masks is None:
return cached_bounds
disc_mask, cup_mask = masks
try:
geom = _geometry_from_mask(disc_mask, self.scale)
except Exception:
return cached_bounds
cx = geom["centre_x"]
cy = geom["centre_y"]
r = geom["crop_radius"]
left = max(0.0, cx - r)
upper = max(0.0, cy - r)
right = min(float(image.width), cx + r)
lower = min(float(image.height), cy + r)
features = compute_geometry_features(disc_mask, cup_mask)
info = {
"left": left,
"upper": upper,
"right": right,
"lower": lower,
"features": features,
}
if cache_path is not None:
np.savez(
cache_path,
left=left,
upper=upper,
right=right,
lower=lower,
width=float(image.width),
height=float(image.height),
scale=self.scale,
target_size=self.target_size,
features=features,
)
return info
def __call__(self, image: Image.Image, image_path: Path) -> Image.Image:
info = self._compute_crop_info(image, image_path)
if info is None:
return image
left = info["left"]
upper = info["upper"]
right = info["right"]
lower = info["lower"]
if right <= left or lower <= upper:
return image
crop = image.crop((left, upper, right, lower))
return crop.resize((self.target_size, self.target_size), Image.BILINEAR)
def geometry_features(self, image: Image.Image, image_path: Path) -> Optional[np.ndarray]:
info = self._compute_crop_info(image, image_path)
if info is None:
return None
features = info.get("features")
if features is None:
return None
return np.asarray(features, dtype=np.float32)
def precompute_geometry(self, image_paths) -> None:
"""Pre-compute geometry features for all image_paths into an in-memory cache.
Safe to call in the main process; geometry_for_image() can then be called
from DataLoader workers without touching CUDA.
"""
self._geometry_cache: Dict[str, Optional[np.ndarray]] = {}
paths = list(image_paths)
print(f"[UNetImageCropper] pre-computing geometry for {len(paths)} images...", flush=True)
for img_path in paths:
key = str(Path(img_path).resolve())
try:
img = Image.open(img_path).convert("RGB")
self._geometry_cache[key] = self.geometry_features(img, img_path)
except Exception:
self._geometry_cache[key] = None
n_ok = sum(1 for v in self._geometry_cache.values() if v is not None)
print(f"[UNetImageCropper] {n_ok}/{len(paths)} geometry vectors computed", flush=True)
def geometry_for_image(self, image_path) -> Optional[np.ndarray]:
"""Return pre-computed geometry vector for image_path (call precompute_geometry first)."""
cache = getattr(self, "_geometry_cache", None)
if cache is None:
raise RuntimeError("Call precompute_geometry() before geometry_for_image()")
return cache.get(str(Path(image_path).resolve()))
class ManifestImageCropper:
def __init__(
self,
manifest_path: Path,
scale: float = 2.5,
target_size: int = 224,
cache_dir: Optional[Path] = None,
) -> None:
self.scale = scale
self.target_size = target_size
self.cache_dir = Path(cache_dir) if cache_dir is not None else None
if self.cache_dir is not None:
self.cache_dir.mkdir(parents=True, exist_ok=True)
df = pd.read_csv(manifest_path)
self.entries: Dict[str, dict] = {}
for _, row in df.iterrows():
img_path = Path(row["image_path"]).resolve()
self.entries[str(img_path)] = {
"annotation_disc": row.get("annotation_disc"),
"annotation_cup": row.get("annotation_cup"),
"annotation_type_disc": row.get("annotation_type_disc"),
"annotation_type_cup": row.get("annotation_type_cup"),
}
def _cache_path(self, image_path: Path) -> Optional[Path]:
if self.cache_dir is None:
return None
return self.cache_dir / f"{image_path.stem}_s{int(self.scale * 100)}.npz"
def clear_cache(self) -> None:
if self.cache_dir is None or not self.cache_dir.exists():
return
removed = sum(1 for f in self.cache_dir.glob("*.npz") if f.unlink() or True)
print(f"[ManifestImageCropper] Cleared {removed} cached crop files from {self.cache_dir}")
@staticmethod
def _load_contour(path: Path) -> np.ndarray:
coords = np.loadtxt(path)
if coords.ndim == 1:
coords = coords.reshape(-1, 2)
return coords
@staticmethod
def _contour_to_mask(coords: np.ndarray, size: tuple[int, int]) -> np.ndarray:
if coords is None or coords.size == 0:
return np.zeros((size[1], size[0]), dtype=np.uint8)
img = Image.new("L", size, 0)
draw = ImageDraw.Draw(img)
points = [tuple(map(float, pt)) for pt in coords]
draw.polygon(points, outline=1, fill=1)
return np.array(img, dtype=np.uint8)
def _load_masks(self, entry: dict, image: Image.Image) -> Optional[Tuple[np.ndarray, np.ndarray]]:
disc_path = entry.get("annotation_disc")
cup_path = entry.get("annotation_cup")
disc_type = (entry.get("annotation_type_disc") or "").lower()
cup_type = (entry.get("annotation_type_cup") or "").lower()
disc_mask: Optional[np.ndarray] = None
cup_mask: Optional[np.ndarray] = None
if disc_path and not pd.isna(disc_path):
disc_path = Path(disc_path)
try:
if disc_type == "mask":
mask_img = Image.open(disc_path)
mask_img = mask_img.resize(image.size, Image.NEAREST)
disc_mask, cup_from_mask = disc_cup_from_mask_image(mask_img)
if cup_from_mask.sum() > 0:
cup_mask = cup_from_mask
elif disc_type == "contour":
coords = self._load_contour(disc_path)
disc_mask = self._contour_to_mask(coords, image.size)
except Exception:
disc_mask = None
if cup_mask is None and cup_path and not pd.isna(cup_path):
cup_path = Path(cup_path)
try:
if cup_type == "mask":
mask_img = Image.open(cup_path)
mask_img = mask_img.resize(image.size, Image.NEAREST)
_, cup_mask = disc_cup_from_mask_image(mask_img)
elif cup_type == "contour":
coords = self._load_contour(cup_path)
cup_mask = self._contour_to_mask(coords, image.size)
except Exception:
cup_mask = None
if disc_mask is None:
return None
disc_mask = (disc_mask > 0).astype(np.uint8)
if cup_mask is None:
cup_mask = np.zeros_like(disc_mask, dtype=np.uint8)
cup_mask = ((cup_mask > 0) & (disc_mask > 0)).astype(np.uint8)
return disc_mask, cup_mask
def _compute_crop_info(self, image: Image.Image, image_path: Path) -> Optional[dict]:
image_path = Path(image_path).resolve()
entry = self.entries.get(str(image_path))
if entry is None:
return None
cache_path = self._cache_path(image_path)
cached_bounds = None
if cache_path is not None and cache_path.exists():
data = np.load(cache_path, allow_pickle=False)
try:
cached_bounds = {
"left": float(data["left"]),
"upper": float(data["upper"]),
"right": float(data["right"]),
"lower": float(data["lower"]),
}
if "features" in data.files:
cached_bounds["features"] = data["features"].astype(np.float32)
return cached_bounds
except KeyError:
cached_bounds = None
masks = self._load_masks(entry, image)
if masks is None:
return cached_bounds
disc_mask, cup_mask = masks
try:
geom = _geometry_from_mask(disc_mask, self.scale)
except Exception:
return cached_bounds
cx = geom["centre_x"]
cy = geom["centre_y"]
r = geom["crop_radius"]
left = max(0.0, cx - r)
upper = max(0.0, cy - r)
right = min(float(image.width), cx + r)
lower = min(float(image.height), cy + r)
features = compute_geometry_features(disc_mask, cup_mask)
info = {
"left": left,
"upper": upper,
"right": right,
"lower": lower,
"features": features,
}
if cache_path is not None:
np.savez(
cache_path,
left=left,
upper=upper,
right=right,
lower=lower,
width=float(image.width),
height=float(image.height),
scale=self.scale,
target_size=self.target_size,
features=features,
)
return info
def __call__(self, image: Image.Image, image_path: Path) -> Image.Image:
info = self._compute_crop_info(image, image_path)
if info is None:
return image
left = info["left"]
upper = info["upper"]
right = info["right"]
lower = info["lower"]
if right <= left or lower <= upper:
return image
crop = image.crop((left, upper, right, lower))
return crop.resize((self.target_size, self.target_size), Image.BILINEAR)
def geometry_features(self, image: Image.Image, image_path: Path) -> Optional[np.ndarray]:
info = self._compute_crop_info(image, image_path)
if info is None:
return None
features = info.get("features")
if features is None:
return None
return np.asarray(features, dtype=np.float32)
def precompute_geometry(self, image_paths) -> None:
"""Pre-compute geometry features for all image_paths into an in-memory cache.
Safe to call in the main process; geometry_for_image() can then be called
without re-opening images or re-loading annotations.
"""
self._geometry_cache: Dict[str, Optional[np.ndarray]] = {}
paths = list(image_paths)
print(f"[ManifestImageCropper] pre-computing geometry for {len(paths)} images...", flush=True)
for img_path in paths:
key = str(Path(img_path).resolve())
entry = self.entries.get(key)
if entry is None:
self._geometry_cache[key] = None
continue
try:
img = Image.open(img_path).convert("RGB")
self._geometry_cache[key] = self.geometry_features(img, img_path)
except Exception:
self._geometry_cache[key] = None
n_ok = sum(1 for v in self._geometry_cache.values() if v is not None)
print(f"[ManifestImageCropper] {n_ok}/{len(paths)} geometry vectors computed", flush=True)
def geometry_for_image(self, image_path) -> Optional[np.ndarray]:
"""Return pre-computed geometry vector for image_path (call precompute_geometry first)."""
cache = getattr(self, "_geometry_cache", None)
if cache is None:
raise RuntimeError("Call precompute_geometry() before geometry_for_image()")
return cache.get(str(Path(image_path).resolve()))
# ---------------------------------------------------------------------------
# Factory
# ---------------------------------------------------------------------------
def build_image_preprocessor_from_args(args):
"""Construct the correct image cropper from CLI args, or return None."""
crop_manifest = getattr(args, "img_crop_manifest", None)
crop_weights = getattr(args, "img_crop_weights", None)
use_gt = bool(getattr(args, "img_crop_gt", False))
if not crop_manifest:
return None
crop_cache = Path(getattr(args, "img_crop_cache", Path("cache_data/hypertower_crops")))
persist_cache = bool(getattr(args, "persist_img_crop_cache", False))
if use_gt:
pre = ManifestImageCropper(
manifest_path=Path(crop_manifest),
scale=getattr(args, "img_crop_scale", 2.5),
target_size=getattr(args, "img_crop_size", 224),
cache_dir=crop_cache,
)
if not persist_cache:
pre.clear_cache()
print(f"[V2 modes] GT disc cropper enabled -> cache at {crop_cache}", flush=True)
return pre
if crop_weights:
pre = UNetImageCropper(
manifest_path=Path(crop_manifest),
weights_path=Path(crop_weights),
normalize=getattr(args, "img_crop_normalize", "per_image"),
threshold=getattr(args, "img_crop_threshold", 0.5),
tta=getattr(args, "img_crop_tta", False),
scale=getattr(args, "img_crop_scale", 2.5),
target_size=getattr(args, "img_crop_size", 224),
cache_dir=crop_cache,
)
if not persist_cache:
pre.clear_cache()
print(f"[V2 modes] UNet disc cropper enabled -> cache at {crop_cache}", flush=True)
return pre
print(
"[V2 modes] img_crop_manifest provided but no --img-crop-gt or --img-crop-weights; cropping disabled.",
flush=True,
)
return None
classes/clinical_data.py → v3/classes/data_bundle.py
Executable → Regular
+88 -111
View File
@@ -1,78 +1,73 @@
# clinical_data.py
from __future__ import annotations
from pathlib import Path
from typing import Iterable, Optional, Dict, List, Tuple
from typing import Callable, Dict, Iterable, List, Optional, Tuple
import numpy as np
import pandas as pd
class ClinicalData:
class DataBundle:
"""
Torch-free container for clinical metadata and file/label bookkeeping.
- Holds one or more dataframes (via add_df) and harmonizes columns
- Canonical IDs: 'Patient ID' must exist (or be specified and will be renamed)
- Canonical eye column: 'eyeID' recoded to 'OS'/'OD' if present; if absent, set to 0
- Feature typing (if cat_cols not provided):
* Categorical if (a) <= max_unique categorical threshold (default 4), or
(b) values cannot be coerced to float; otherwise numeric (scalar)
- Scaling/imputation:
* Numeric: min–max to [0,1], median imputation; + one missing flag per numeric feature
* Categorical: one-hot with '<UNK>' bucket at index 0
- Patient-level K-fold indices stored as dict: folds[k] -> {'train_ids': [...], 'test_ids': [...]}
Generic, torch-free container for metadata and file/label bookkeeping.
Keeps feature typing, vectorization, and patient-level splits generic.
Dataset-specific preprocessing (e.g., eye canonicalization) should live
in the dataset builder (e.g., papila_builders in v2).
"""
def __init__(
self,
*,
image_dir: str,
clinical_dir: Optional[str],
clinical_dir: Optional[str] = None,
label_col: str,
# typing / detection
patient_col: str = "Patient ID",
cat_cols: Optional[Iterable[str]] = None,
max_unique_for_cat: int = 4,
# splitting
n_splits: int = 5,
random_seed: int = 42,
):
filename_template: str = "RET{pid:03d}{eye}.jpg",
image_path_fn: Optional[Callable[[pd.Series], Path]] = None,
) -> None:
self.image_dir = Path(image_dir)
self.clinical_dir = Path(clinical_dir) if clinical_dir else None
self.label_col = label_col
self.patient_col = patient_col
self.max_unique_for_cat = max_unique_for_cat
self.n_splits = n_splits
self.filename_template = filename_template
self.image_path_fn = image_path_fn
self.clinical_dir = Path(clinical_dir) if clinical_dir else None
# Internal state
self.frames: List[pd.DataFrame] = [] # raw frames as added
self.df: pd.DataFrame = pd.DataFrame() # concatenated
self.frames: List[pd.DataFrame] = []
self.df: pd.DataFrame = pd.DataFrame()
self.scalar_cols: List[str] = []
self.cat_cols: List[str] = list(cat_cols) if cat_cols is not None else []
self.scalar_stats: Dict[str, Dict[str, float]] = {}
self.cat_maps: Dict[str, Dict[object, int]] = {}
self.feature_dim: int = 0
self.folds: Dict[int, Dict[str, List[object]]] = {} # fold -> {'train_ids': [], 'test_ids': []}
self.folds: Dict[int, Dict[str, List[object]]] = {}
self.random_seed = int(random_seed)
# ------------------- Public API -------------------
def add_df(
self,
df: pd.DataFrame,
*,
id_column: Optional[str] = None,
eye_column: Optional[str] = None,
exclude_cols: Optional[Iterable[str]] = None,
) -> None:
"""
Add a dataframe and re-run harmonization, typing, stats, and K-fold indices.
Add a dataframe and re-run typing, stats, and K-fold indices.
QC rules:
- Must have patient ID column; if not provided under that name, specify id_column.
- eyeID, if present, must be binary; recoded to 'OS'/'OD'. If absent, create and set to 0.
"""
df = df.copy()
# --- QC: Patient ID ---
pid_col = self._ensure_patient_id(df, id_column)
# --- QC: eyeID ---
self._canonicalize_eye_column(df, eye_column)
# --- Normalize label presence ---
self._ensure_patient_id(df, id_column)
if self.label_col not in df.columns:
raise ValueError(f"label_col '{self.label_col}' not found in added dataframe")
# append & refresh
self.frames.append(df)
self._refresh_master_df(exclude_cols=exclude_cols)
self._infer_or_validate_feature_types(exclude_cols=exclude_cols)
@@ -83,13 +78,14 @@ class ClinicalData:
def get_split_ids(self, fold: int) -> Tuple[List[object], List[object]]:
rec = self.folds.get(fold)
if not rec: raise KeyError(f"Fold {fold} not available. Built folds: {sorted(self.folds.keys())}")
return rec['train_ids'], rec['test_ids']
if not rec:
raise KeyError(f"Fold {fold} not available. Built folds: {sorted(self.folds.keys())}")
return rec["train_ids"], rec["test_ids"]
def get_split_dfs(self, fold: int) -> Tuple[pd.DataFrame, pd.DataFrame]:
train_ids, test_ids = self.get_split_ids(fold)
train_df = self.df[self.df['Patient ID'].isin(train_ids)].reset_index(drop=True)
test_df = self.df[self.df['Patient ID'].isin(test_ids)].reset_index(drop=True)
train_df = self.df[self.df[self.patient_col].isin(train_ids)].reset_index(drop=True)
test_df = self.df[self.df[self.patient_col].isin(test_ids)].reset_index(drop=True)
return train_df, test_df
def vectorize_row(self, row: pd.Series) -> np.ndarray:
@@ -98,13 +94,14 @@ class ClinicalData:
miss: List[float] = []
# numeric
for col in self.scalar_cols:
v = pd.to_numeric(row.get(col), errors='coerce')
v = pd.to_numeric(row.get(col), errors="coerce")
if pd.isna(v):
miss.append(1.0)
v = self.scalar_stats[col]['median']
v = self.scalar_stats[col]["median"]
else:
miss.append(0.0)
lo = self.scalar_stats[col]['min']; hi = self.scalar_stats[col]['max']
lo = self.scalar_stats[col]["min"]
hi = self.scalar_stats[col]["max"]
feats.append((float(v) - lo) / (hi - lo) if hi > lo else 0.0)
# categorical
for col in self.cat_cols:
@@ -117,89 +114,62 @@ class ClinicalData:
feats.extend(miss)
return np.asarray(feats, dtype=np.float32)
def get_image_path(self, row: pd.Series, filename_template: str = "RET{pid:03d}{eye}.jpg") -> Path:
pid = int(row['Patient ID']); eye = row.get('eyeID', 0)
def get_image_path(self, row: pd.Series) -> Path:
if self.image_path_fn is not None:
return Path(self.image_path_fn(row))
pid = int(row[self.patient_col])
eye = row.get("eyeID", "")
if eye in ("OS", "OD"):
eye_str = eye
else:
eye_str = str(eye)
return self.image_dir / filename_template.format(pid=pid, eye=eye_str)
return self.image_dir / self.filename_template.format(pid=pid, eye=eye_str)
def encode_metadata(self, row: pd.Series) -> np.ndarray:
return self.vectorize_row(row)
def get_label(self, row: pd.Series) -> int:
return int(row[self.label_col])
# ------------------- Internal helpers -------------------
def _ensure_patient_id(self, df: pd.DataFrame, id_column: Optional[str]) -> str:
if 'Patient ID' in df.columns:
return 'Patient ID'
if id_column and id_column in df.columns:
df.rename(columns={id_column: 'Patient ID'}, inplace=True)
return 'Patient ID'
# try auto-detect common variants
candidates = [c for c in df.columns if c.lower().replace(" ", "") in {"patientid","patient","pid"}]
if len(candidates) == 1:
df.rename(columns={candidates[0]: 'Patient ID'}, inplace=True)
return 'Patient ID'
raise ValueError("A 'Patient ID' column is required; provide id_column=... if it has a different name.")
def _canonicalize_eye_column(self, df: pd.DataFrame, eye_column: Optional[str]) -> None:
# Find source
src = None
if 'eyeID' in df.columns: src = 'eyeID'
elif eye_column and eye_column in df.columns: src = eye_column
else:
# try auto detect
for c in df.columns:
if 'eye' in c.lower():
src = c; break
if src is None:
df['eyeID'] = 0
def _ensure_patient_id(self, df: pd.DataFrame, id_column: Optional[str]) -> None:
if self.patient_col in df.columns:
return
# Map to OS/OD
s = df[src]
def norm(v):
if pd.isna(v): return None
x = str(v).strip().upper()
if x in {"OS","L","LEFT","0"}: return "OS"
if x in {"OD","R","RIGHT","1"}: return "OD"
# numbers like 2? fall back by parity
try:
num = int(float(x))
return "OD" if num % 2 == 1 else "OS"
except Exception:
return None
mapped = s.map(norm)
uniq = {u for u in mapped.dropna().unique().tolist()}
if not uniq.issubset({"OS","OD"}):
raise ValueError(f"eyeID must be binary; found values {sorted(uniq)}")
df['eyeID'] = mapped.fillna("OS")
if src != 'eyeID':
# keep original too if you want, but we standardize on 'eyeID'
pass
if id_column and id_column in df.columns:
df.rename(columns={id_column: self.patient_col}, inplace=True)
return
candidates = [
c
for c in df.columns
if c.lower().replace(" ", "") in {"patientid", "patient", "pid"}
]
if len(candidates) == 1:
df.rename(columns={candidates[0]: self.patient_col}, inplace=True)
return
raise ValueError(
f"A '{self.patient_col}' column is required; provide id_column=... if it has a different name."
)
def _refresh_master_df(self, exclude_cols: Optional[Iterable[str]] = None) -> None:
self.df = pd.concat(self.frames, axis=0, ignore_index=True)
# drop columns explicitly excluded
if exclude_cols:
self.df = self.df.drop(columns=[c for c in exclude_cols if c in self.df.columns])
def _infer_or_validate_feature_types(self, exclude_cols: Optional[Iterable[str]] = None) -> None:
excluded = set(exclude_cols or []) | {self.label_col, 'Patient ID'}
# we keep canonical 'eyeID' as categorical if present
excluded = set(exclude_cols or []) | {self.label_col, self.patient_col}
feature_candidates = [c for c in self.df.columns if c not in excluded]
# If user pre-specified cat_cols in __init__, respect them and infer the rest
cats = set(self.cat_cols) if self.cat_cols else set()
scalars = set()
for c in feature_candidates:
if c == 'eyeID':
cats.add('eyeID'); continue
if c in cats: continue
if c in cats:
continue
s = self.df[c]
# try numeric coercion
as_num = pd.to_numeric(s, errors='coerce')
as_num = pd.to_numeric(s, errors="coerce")
num_missing = as_num.isna().mean()
num_unique = s.dropna().nunique()
if as_num.notna().any() and num_missing < 1.0 and num_unique > self.max_unique_for_cat:
scalars.add(c)
else:
# categorical if few uniques OR non-numeric
if num_unique <= self.max_unique_for_cat or as_num.isna().mean() > 0.0:
cats.add(c)
else:
@@ -210,24 +180,28 @@ class ClinicalData:
def _compute_numeric_stats(self) -> None:
self.scalar_stats.clear()
for col in self.scalar_cols:
s = pd.to_numeric(self.df[col], errors='coerce')
s = pd.to_numeric(self.df[col], errors="coerce")
vals = s.dropna().astype(float).values
if vals.size == 0:
lo, hi, med = 0.0, 1.0, 0.0
else:
lo, hi = float(np.min(vals)), float(np.max(vals))
med = float(np.median(vals))
if hi <= lo: hi = lo + 1.0
if hi <= lo:
hi = lo + 1.0
self.scalar_stats[col] = {"min": lo, "max": hi, "median": med}
def _build_cat_maps(self) -> None:
self.cat_maps.clear()
for col in self.cat_cols:
cats = [v for v in self.df[col].dropna().unique().tolist()]
try: cats = sorted(cats)
except Exception: pass
try:
cats = sorted(cats)
except Exception:
pass
mapping = {"<UNK>": 0}
for i, v in enumerate(cats, start=1): mapping[v] = i
for i, v in enumerate(cats, start=1):
mapping[v] = i
self.cat_maps[col] = mapping
def _compute_feature_dim(self) -> None:
@@ -235,11 +209,9 @@ class ClinicalData:
# ------------------- K-fold on unique patients -------------------
def _build_kfold_indices(self) -> None:
# unique patients and a per-patient label for stratification if possible
pats = self.df['Patient ID'].unique().tolist()
# Derive a patient label as the mode of their rows (fallback to first valid)
labels_by_pat = {}
for pid, grp in self.df.groupby('Patient ID'):
pats = self.df[self.patient_col].unique().tolist()
labels_by_pat: Dict[object, object] = {}
for pid, grp in self.df.groupby(self.patient_col):
lab = grp[self.label_col].dropna()
if len(lab) == 0:
labels_by_pat[pid] = 0
@@ -247,18 +219,23 @@ class ClinicalData:
labels_by_pat[pid] = lab.mode().iloc[0]
y_pat = np.array([labels_by_pat[p] for p in pats])
# Try to use StratifiedGroupKFold if available, else fall back to StratifiedKFold on patient labels
try:
from sklearn.model_selection import StratifiedGroupKFold
sgkf = StratifiedGroupKFold(n_splits=self.n_splits, shuffle=True, random_state=self.random_seed)
sgkf = StratifiedGroupKFold(
n_splits=self.n_splits, shuffle=True, random_state=self.random_seed
)
split_iter = sgkf.split(X=pats, y=y_pat, groups=pats)
except Exception:
from sklearn.model_selection import StratifiedKFold
skf = StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=self.random_seed)
skf = StratifiedKFold(
n_splits=self.n_splits, shuffle=True, random_state=self.random_seed
)
split_iter = skf.split(X=np.zeros(len(pats)), y=y_pat)
self.folds.clear()
for i, (train_idx, test_idx) in enumerate(split_iter):
train_ids = [pats[j] for j in train_idx]
test_ids = [pats[j] for j in test_idx]
test_ids = [pats[j] for j in test_idx]
self.folds[i] = {"train_ids": train_ids, "test_ids": test_ids}
v3/classes/geometry_features.py
Executable
+87
View File
@@ -0,0 +1,87 @@
"""Shared helpers for deriving disc/cup geometry features."""
from __future__ import annotations
from collections import Counter
from typing import Tuple
import numpy as np
from PIL import Image
EPS = 1e-6
FEATURE_DIM = 5
def disc_cup_from_mask_image(mask_img: Image.Image) -> Tuple[np.ndarray, np.ndarray]:
"""Return binary disc/cup masks from a REFUGE-style annotation image."""
arr = np.asarray(mask_img)
if arr.ndim == 3:
h, w, c = arr.shape
border = np.concatenate(
[arr[0, :, :], arr[-1, :, :], arr[:, 0, :], arr[:, -1, :]],
axis=0,
)
border_counts = Counter(map(tuple, border))
bg_color = border_counts.most_common(1)[0][0]
flat = arr.reshape(-1, c)
colors = Counter(map(tuple, flat))
colors.pop(bg_color, None)
disc = (~np.all(arr == bg_color, axis=-1)).astype(np.uint8)
if colors:
cup_color = min(colors.keys(), key=lambda col: sum(col))
cup = np.all(arr == cup_color, axis=-1).astype(np.uint8)
else:
cup = np.zeros((h, w), dtype=np.uint8)
else:
border = np.concatenate([arr[0, :], arr[-1, :], arr[:, 0], arr[:, -1]])
counts = Counter(border.tolist())
bg_value = counts.most_common(1)[0][0]
disc = (arr != bg_value).astype(np.uint8)
fg = arr[arr != bg_value]
if fg.size > 0:
cup_value = int(np.min(fg))
cup = (arr == cup_value).astype(np.uint8)
else:
cup = np.zeros_like(arr, dtype=np.uint8)
cup = (cup > 0) & (disc > 0)
return disc.astype(np.uint8), cup.astype(np.uint8)
def compute_geometry_features(disc_mask: np.ndarray, cup_mask: np.ndarray) -> np.ndarray:
"""Compute cup/disc geometry descriptors (area, rim, diameter ratios, centre shift)."""
disc = (disc_mask > 0).astype(np.float32)
cup = (cup_mask > 0).astype(np.float32)
disc_area = disc.sum()
cup_area = cup.sum()
area_ratio = cup_area / (disc_area + EPS)
rim_ratio = (disc_area - cup_area) / (disc_area + EPS)
disc_rows = np.any(disc > 0, axis=1)
cup_rows = np.any(cup > 0, axis=1)
disc_cols = np.any(disc > 0, axis=0)
cup_cols = np.any(cup > 0, axis=0)
disc_height = float(disc_rows.sum())
cup_height = float(cup_rows.sum())
disc_width = float(disc_cols.sum())
cup_width = float(cup_cols.sum())
vertical_ratio = cup_height / (disc_height + EPS)
horizontal_ratio = cup_width / (disc_width + EPS)
def _centre(mask: np.ndarray) -> Tuple[float, float]:
coords = np.argwhere(mask > 0)
if coords.size == 0:
return 0.5, 0.5
ys, xs = coords[:, 0], coords[:, 1]
return float(xs.mean()) / mask.shape[1], float(ys.mean()) / mask.shape[0]
disc_cx, disc_cy = _centre(disc)
cup_cx, cup_cy = _centre(cup)
centre_shift = float(np.hypot(cup_cx - disc_cx, cup_cy - disc_cy))
return np.array(
[area_ratio, rim_ratio, vertical_ratio, horizontal_ratio, centre_shift],
dtype=np.float32,
)
v3/classes/geometry_towers.py
+859
View File
@@ -0,0 +1,859 @@
"""geometry_towers — GeometryTower and all segmentation-map infrastructure.
Self-contained: absorbs everything that was in seg_cnn.py so that file can
eventually be removed. Does not import from seg_cnn.py or any other tower file.
Imports only TowerBase from towerbase plus standard infrastructure.
Contents
--------
SegMapRecord — labelled-eye data record
_combine_masks — merge disc/cup binary masks → 3-class label map
crop_to_disc — tight bounding-box crop
seg_map_to_tensor — (H,W) uint8 → (C,H,W) float32 tensor
load_gt_masks — load GT disc+cup masks from contour/mask files
UNetFineTuneDataset — Dataset for fine-tuning the UNet on GT annotations
precompute_unet_seg_maps — batch UNet inference helper
SegMapDataset — Dataset yielding (seg_tensor, label) pairs
SegCNN — pretrained CNN adapted for segmentation-map input
GeometryTower — TowerBase implementation (the main class to use)
"""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import List, Optional, Tuple
import numpy as np
import torch
import torch.nn as nn
from PIL import Image, ImageDraw
from PIL.Image import Resampling
from torch.utils.data import Dataset
from torchvision import models, transforms
from tqdm import tqdm
import pandas as pd
from v3.classes.towerbase import TowerBase
from v3.classes.towerbase import TowerBase, EarlyPassContext
# ---------------------------------------------------------------------------
# Data record
# ---------------------------------------------------------------------------
@dataclass
class SegMapRecord:
"""One labelled eye sample for the seg-map CNN."""
sample_id: str
image_path: Path # original fundus image (used by unet mode)
annotation_disc: Path # contour (.txt) or mask (.bmp/.png)
annotation_cup: Path
annotation_type_disc: str # "contour" or "mask"
annotation_type_cup: str
patient_id: int # for group-CV: keep both eyes of a patient together
eye: str # "OD" or "OS"
label: int # 0 = Normal, 1 = Glaucoma
# ---------------------------------------------------------------------------
# Seg-map utilities
# ---------------------------------------------------------------------------
def _combine_masks(disc_mask: np.ndarray, cup_mask: np.ndarray) -> np.ndarray:
"""Combine binary disc and cup masks into a 3-class label map.
Returns a uint8 array with values:
0 — background
1 — optic disc rim (disc but not cup)
2 — optic cup
"""
disc = (disc_mask > 0).astype(np.uint8)
cup = (cup_mask > 0).astype(np.uint8)
cup = cup & disc # structural prior: cup must be inside disc
seg = disc + cup # 0, 1 (rim), or 2 (cup)
return seg.astype(np.uint8)
def crop_to_disc(seg_map: np.ndarray) -> np.ndarray:
"""Crop a seg map tightly to the disc bounding box.
The disc is anywhere seg_map > 0 (i.e. rim or cup).
Returns the original array unchanged if no disc is found.
"""
rows = np.any(seg_map > 0, axis=1)
cols = np.any(seg_map > 0, axis=0)
if not rows.any():
return seg_map
r0, r1 = int(np.argmax(rows)), int(len(rows) - 1 - np.argmax(rows[::-1]))
c0, c1 = int(np.argmax(cols)), int(len(cols) - 1 - np.argmax(cols[::-1]))
return seg_map[r0 : r1 + 1, c0 : c1 + 1]
def seg_map_to_tensor(
seg_map: np.ndarray,
channels: int,
target_size: int,
) -> torch.Tensor:
"""Convert an (H, W) seg map with values {0, 1, 2} to a float tensor.
channels=1 → (1, H, W) float in [0, 1] (values 0/0.5/1.0)
channels=3 → (3, H, W) one-hot binary channels [bg, disc_rim, cup]
"""
pil = Image.fromarray(seg_map.astype(np.uint8), mode="L")
pil = pil.resize((target_size, target_size), Image.NEAREST)
seg = np.array(pil, dtype=np.uint8)
if channels == 1:
arr = seg.astype(np.float32) / 2.0 # {0, 0.5, 1.0}
return torch.from_numpy(arr).unsqueeze(0)
if channels == 3:
bg = (seg == 0).astype(np.float32)
disc_rim = (seg == 1).astype(np.float32)
cup = (seg == 2).astype(np.float32)
return torch.from_numpy(np.stack([bg, disc_rim, cup], axis=0))
raise ValueError(f"channels must be 1 or 3, got {channels}")
# ---------------------------------------------------------------------------
# GT mask loading (pure NumPy / PIL — no CUDA, safe in DataLoader workers)
# ---------------------------------------------------------------------------
def _load_contour(path: Path) -> np.ndarray:
"""Load x,y contour pairs from a whitespace- or comma-delimited text file."""
arr = np.zeros((0, 2), dtype=np.float32)
for delimiter in (",", None):
try:
candidate = np.loadtxt(
str(path), delimiter=delimiter, comments="#", dtype=np.float32
)
if candidate.size > 0:
arr = candidate
break
except Exception:
pass
if arr.size == 0 or arr.ndim == 1:
return np.zeros((0, 2), dtype=np.float32)
if arr.shape[1] < 2:
return np.zeros((0, 2), dtype=np.float32)
return arr[:, :2]
def _contour_to_mask(
coords: np.ndarray, image_size: Tuple[int, int], target_size: int
) -> np.ndarray:
"""Rasterise a polygon defined by (x, y) coords into a binary mask.
image_size is the (width, height) of the original fundus image — the
coordinate space the contour was annotated in.
"""
if coords is None or len(coords) < 3:
return np.zeros((target_size, target_size), dtype=np.uint8)
points = [tuple(map(float, pt)) for pt in coords]
img = Image.new("L", image_size, 0)
ImageDraw.Draw(img).polygon(points, outline=1, fill=1)
img = img.resize((target_size, target_size), Resampling.NEAREST)
return (np.array(img, dtype=np.uint8) > 0).astype(np.uint8)
def _extract_masks_from_image(
mask_path: Path, target_size: int
) -> Tuple[np.ndarray, np.ndarray]:
"""Extract disc and cup binary masks from a segmentation image file.
Handles both grayscale label images (e.g. REFUGE .bmp) and
RGB colour-coded masks. Returns (disc_mask, cup_mask) both at
target_size × target_size.
"""
raw = Image.open(mask_path)
arr = np.array(raw)
if arr.ndim == 2:
edges = np.concatenate([arr[0], arr[-1], arr[:, 0], arr[:, -1]])
bg_val = int(
np.argmax(np.bincount(edges.astype(np.int64).clip(0, 255), minlength=256))
)
disc_arr = (arr != bg_val).astype(np.uint8)
vals = np.unique(arr)
non_bg = vals[vals != bg_val]
cup_arr: np.ndarray
if non_bg.size > 1:
cup_val = int(non_bg.min())
cup_arr = (arr == cup_val).astype(np.uint8)
else:
cup_arr = np.zeros_like(disc_arr, dtype=np.uint8)
else:
img_rgb = raw.convert("RGB")
arr = np.array(img_rgb)
h, w, c = arr.shape
edges_rgb = np.concatenate([arr[0], arr[-1], arr[:, 0], arr[:, -1]], axis=0)
edge_colors, edge_counts = np.unique(
edges_rgb.reshape(-1, c), axis=0, return_counts=True
)
bg_color = edge_colors[int(np.argmax(edge_counts))]
colors, counts = np.unique(arr.reshape(-1, c), axis=0, return_counts=True)
not_bg = np.any(colors != bg_color.reshape(1, -1), axis=1)
colors, counts = colors[not_bg], counts[not_bg]
disc_arr = np.zeros((h, w), dtype=np.uint8)
cup_arr = np.zeros((h, w), dtype=np.uint8)
if colors.shape[0] >= 1:
order = np.argsort(-counts)
disc_color = colors[order[0]]
disc_arr[np.all(arr == disc_color, axis=-1)] = 1
if colors.shape[0] >= 2:
cup_color = colors[order[1]]
cup_arr[np.all(arr == cup_color, axis=-1)] = 1
def _resize(m: np.ndarray) -> np.ndarray:
pil = Image.fromarray((m > 0).astype(np.uint8) * 255)
pil = pil.resize((target_size, target_size), Resampling.NEAREST)
return (np.array(pil) > 0).astype(np.uint8)
return _resize(disc_arr), _resize(cup_arr)
def load_gt_masks(rec: SegMapRecord, target_size: int) -> Tuple[np.ndarray, np.ndarray]:
"""Load GT disc + cup masks for one record.
Handles annotation_type "contour" (x,y text file) and "mask" (image file).
Returns (disc_mask, cup_mask) as uint8 arrays of shape (target_size, target_size).
"""
disc_mask: Optional[np.ndarray] = None
cup_mask: Optional[np.ndarray] = None
with Image.open(rec.image_path) as _img:
image_size = _img.size # (width, height)
# ---- Disc ----
if rec.annotation_type_disc == "mask":
disc_mask, cup_from_disc = _extract_masks_from_image(
rec.annotation_disc, target_size
)
if cup_from_disc.any():
cup_mask = cup_from_disc
else: # contour
coords = _load_contour(rec.annotation_disc)
disc_mask = _contour_to_mask(coords, image_size, target_size)
# ---- Cup ----
if cup_mask is None:
if rec.annotation_type_cup == "mask":
_, cup_from_cup = _extract_masks_from_image(rec.annotation_cup, target_size)
cup_mask = cup_from_cup
else: # contour
coords = _load_contour(rec.annotation_cup)
cup_mask = _contour_to_mask(coords, image_size, target_size)
if disc_mask is None:
disc_mask = np.zeros((target_size, target_size), dtype=np.uint8)
if cup_mask is None:
cup_mask = np.zeros((target_size, target_size), dtype=np.uint8)
cup_mask = (cup_mask > 0) & (disc_mask > 0)
return disc_mask.astype(np.uint8), cup_mask.astype(np.uint8)
# ---------------------------------------------------------------------------
# U-Net fine-tuning dataset
# ---------------------------------------------------------------------------
class UNetFineTuneDataset(Dataset):
"""Loads (image_tensor, mask_tensor) pairs for fine-tuning the U-Net."""
def __init__(
self,
records: List[SegMapRecord],
target_size: int = 512,
normalize: str = "per_image",
) -> None:
self.records = records
self.target_size = target_size
self.normalize = normalize
self.to_tensor = transforms.ToTensor()
def __len__(self) -> int:
return len(self.records)
def _normalize(self, tensor: torch.Tensor) -> torch.Tensor:
if self.normalize == "per_image":
mean = tensor.mean(dim=(1, 2), keepdim=True)
std = tensor.std(dim=(1, 2), keepdim=True).clamp(min=1e-6)
return (tensor - mean) / std
if self.normalize == "imagenet":
mean = torch.tensor([0.485, 0.456, 0.406]).view(-1, 1, 1)
std = torch.tensor([0.229, 0.224, 0.225]).view(-1, 1, 1)
return (tensor - mean) / std
return tensor
def __getitem__(self, idx: int):
rec = self.records[idx]
image = Image.open(rec.image_path).convert("RGB")
image = image.resize((self.target_size, self.target_size), Resampling.BILINEAR)
img_tensor = self._normalize(self.to_tensor(image))
disc_mask, cup_mask = load_gt_masks(rec, self.target_size)
mask_tensor = torch.from_numpy(
np.stack([disc_mask, cup_mask], axis=0).astype(np.float32)
)
return img_tensor, mask_tensor
# ---------------------------------------------------------------------------
# U-Net precomputation
# ---------------------------------------------------------------------------
def precompute_unet_seg_maps(
records: List[SegMapRecord],
segmenter,
threshold: float = 0.5,
) -> List[np.ndarray]:
"""Run the U-Net on every record and return a list of combined seg maps."""
to_tensor = transforms.ToTensor()
seg_maps = []
for rec in tqdm(records, desc="U-Net inference", unit="img", leave=False):
image = Image.open(rec.image_path).convert("RGB")
resized = segmenter.preprocess_image(image)
tensor = segmenter._normalize_tensor(
to_tensor(resized).to(segmenter.device)
).unsqueeze(0)
with torch.no_grad():
logits = segmenter.model(tensor)
probs = torch.sigmoid(logits)[0].cpu().numpy()
disc = (probs[0] > threshold).astype(np.uint8)
cup = (probs[1] > threshold).astype(np.uint8)
cup = cup & disc
seg_maps.append(_combine_masks(disc, cup.astype(np.uint8)))
return seg_maps
# ---------------------------------------------------------------------------
# SegMapDataset
# ---------------------------------------------------------------------------
class SegMapDataset(Dataset):
"""PyTorch Dataset that yields (seg_tensor, label) pairs."""
def __init__(
self,
records: List[SegMapRecord],
target_size: int = 224,
channels: int = 3,
augment: bool = False,
unet_segmenter=None,
unet_threshold: float = 0.5,
seg_target_size: int = 512,
crop_to_disc_flag: bool = True,
precomputed_seg_maps: Optional[List[np.ndarray]] = None,
) -> None:
self.records = records
self.target_size = target_size
self.channels = channels
self.augment = augment
self.seg_target_size = seg_target_size
self.crop_to_disc_flag = crop_to_disc_flag
if precomputed_seg_maps is not None:
self._seg_maps = precomputed_seg_maps
elif unet_segmenter is not None:
self._seg_maps = precompute_unet_seg_maps(
records, unet_segmenter, unet_threshold
)
else:
self._seg_maps = None
def __len__(self) -> int:
return len(self.records)
def _augment(self, seg_map: np.ndarray) -> np.ndarray:
if np.random.rand() < 0.5:
seg_map = np.fliplr(seg_map)
if np.random.rand() < 0.5:
seg_map = np.flipud(seg_map)
k = np.random.randint(0, 4)
if k:
seg_map = np.rot90(seg_map, k=k)
return np.ascontiguousarray(seg_map)
def __getitem__(self, idx: int):
rec = self.records[idx]
if self._seg_maps is not None:
seg_map = self._seg_maps[idx]
else:
disc_mask, cup_mask = load_gt_masks(rec, self.seg_target_size)
seg_map = _combine_masks(disc_mask, cup_mask)
if self.crop_to_disc_flag:
seg_map = crop_to_disc(seg_map)
if self.augment:
seg_map = self._augment(seg_map)
tensor = seg_map_to_tensor(seg_map, self.channels, self.target_size)
return tensor, rec.label
# ---------------------------------------------------------------------------
# SegCNN
# ---------------------------------------------------------------------------
_SEGCNN_FEAT_DIM = {
"resnet18": 512,
"resnet50": 2048,
"efficientnet_b0": 1280,
}
class SegCNN(nn.Module):
"""Pretrained CNN backbone adapted for segmentation-map input.
Parameters
----------
num_classes : output classes (2 for binary glaucoma grading)
backbone : "resnet18" | "resnet50" | "efficientnet_b0"
pretrained : initialise with ImageNet weights
in_channels : 1 (single label map) or 3 (one-hot channels)
dropout : dropout rate before the final classifier head
"""
def __init__(
self,
num_classes: int = 2,
backbone: str = "resnet18",
pretrained: bool = True,
in_channels: int = 3,
dropout: float = 0.3,
) -> None:
super().__init__()
weights_arg = "DEFAULT" if pretrained else None
if backbone == "resnet18":
base = models.resnet18(weights=weights_arg)
feat_dim = base.fc.in_features
base.fc = nn.Identity()
elif backbone == "resnet50":
base = models.resnet50(weights=weights_arg)
feat_dim = base.fc.in_features
base.fc = nn.Identity()
elif backbone == "efficientnet_b0":
base = models.efficientnet_b0(weights=weights_arg)
feat_dim = base.classifier[1].in_features
base.classifier = nn.Identity()
else:
raise ValueError(f"Unknown backbone: {backbone!r}")
if in_channels != 3:
first_conv = self._find_first_conv(base)
new_conv = nn.Conv2d(
in_channels,
first_conv.out_channels,
kernel_size=first_conv.kernel_size,
stride=first_conv.stride,
padding=first_conv.padding,
bias=first_conv.bias is not None,
)
if pretrained:
with torch.no_grad():
new_conv.weight.copy_(
first_conv.weight.mean(dim=1, keepdim=True).expand_as(
new_conv.weight
)
)
self._replace_first_conv(base, new_conv)
self.backbone = base
self.head = nn.Sequential(
nn.Dropout(p=dropout),
nn.Linear(feat_dim, num_classes),
)
@staticmethod
def _find_first_conv(module: nn.Module) -> nn.Conv2d:
for m in module.modules():
if isinstance(m, nn.Conv2d):
return m
raise RuntimeError("No Conv2d found in backbone")
@staticmethod
def _replace_first_conv(module: nn.Module, new_conv: nn.Conv2d) -> None:
for name, child in module.named_children():
if isinstance(child, nn.Conv2d):
setattr(module, name, new_conv)
return
try:
SegCNN._replace_first_conv(child, new_conv)
return
except RuntimeError:
pass
raise RuntimeError("Could not replace first Conv2d")
def forward(self, x: torch.Tensor) -> torch.Tensor:
feats = self.backbone(x)
if feats.dim() > 2:
feats = feats.flatten(1)
return self.head(feats)
# ---------------------------------------------------------------------------
# GeometryTower — TowerBase implementation
# ---------------------------------------------------------------------------
class GeometryTower(TowerBase, nn.Module):
"""TowerBase implementation for the optic-disc/cup segmentation modality.
Encodes a 3-class disc/cup segmentation map (bg=0, rim=1, cup=2) through a
CNN backbone, contributing one spatial embedding to the bridge.
The seg map is produced from GT annotations (manifest-based) or from a
trained U-Net, depending on ``geometry_source``.
``prepare_fold`` builds a seg-map generator, pre-computes all maps for the
fold, and caches them keyed by image path. ``augment_samples`` then
injects ``seg_map_1`` / ``seg_map_2`` float32 numpy arrays (shape C×H×W)
into each sample dict so the DataLoader delivers them as tensors to
``embed_batch``.
Parameters
----------
backbone : CNN backbone — "resnet18" | "resnet50" | "efficientnet_b0"
in_channels : 1 (label map) or 3 (one-hot disc/rim/cup channels)
pretrained : initialise backbone with ImageNet weights
frozen : if True, backbone is always frozen
target_size : spatial size the seg map tensor is resized to
seg_target_size : resolution at which GT masks are rasterised / U-Net runs
crop_to_disc : crop seg map tightly to disc bounding box before resizing
geometry_source : "gt" (manifest annotations) or "unet" (U-Net predictions)
manifest_path : path to the geometry manifest CSV (required)
weights_path : path to UNet checkpoint (required when source="unet")
unet_normalize : UNet normalisation mode (default "per_image")
unet_threshold : UNet mask threshold (default 0.5)
finetune_unet_epochs : epochs to fine-tune U-Net per fold (0 = disabled)
finetune_unet_lr : learning rate for U-Net fine-tuning
"""
def __init__(
self,
*,
backbone: str = "resnet18",
in_channels: int = 3,
pretrained: bool = True,
frozen: bool = False,
target_size: int = 224,
seg_target_size: int = 512,
crop_to_disc: bool = True,
geometry_source: str = "gt",
manifest_path=None,
weights_path=None,
unet_normalize: str = "per_image",
unet_threshold: float = 0.5,
finetune_unet_epochs: int = 0,
finetune_unet_lr: float = 1e-5,
):
nn.Module.__init__(self)
self._backbone_name = backbone
self._in_channels = in_channels
self._frozen = frozen
self._target_size = target_size
self._seg_target_size = seg_target_size
self._crop_to_disc = crop_to_disc
self._geometry_source = geometry_source
self._manifest_path = Path(manifest_path) if manifest_path is not None else None
self._weights_path = Path(weights_path) if weights_path is not None else None
self._unet_normalize = unet_normalize
self._unet_threshold = unet_threshold
self._finetune_unet_epochs = finetune_unet_epochs
self._finetune_unet_lr = finetune_unet_lr
self._out_dim = _SEGCNN_FEAT_DIM.get(backbone, 512)
self._seg_cnn = SegCNN(
num_classes=2,
backbone=backbone,
pretrained=pretrained,
in_channels=in_channels,
)
self._seg_cache: dict = {} # image_path_str → float32 (C, H, W) numpy array
# ------------------------------------------------------------------
# TowerBase interface
# ------------------------------------------------------------------
@property
def embed_dims(self) -> list[int]:
return [self._out_dim]
@property
def total_epochs(self) -> int:
return 0 if self._frozen else 1
def set_phase(self, phase: str) -> None:
trainable = not self._frozen and phase in ("tower_warmup", "main")
for p in self._seg_cnn.parameters():
p.requires_grad = trainable
def early_pass(self, context: EarlyPassContext) -> None:
"""
V4 orchestrator hook: combines prepare_fold and augment_samples.
1. Pre-computes all segmentation maps for the current fold.
2. Injects them into the sample dicts held by the context object.
"""
if self._manifest_path is None:
raise ValueError("GeometryTower requires manifest_path")
all_paths: dict = {}
for split in (
context.eye_train,
context.bilat_train,
context.bilat_val,
context.bilat_test,
):
for s in split:
for slot in ("image_1", "image_2"):
p = s.get(slot)
if p is not None:
all_paths[str(Path(p).resolve())] = None
# 1. Pre-compute seg maps (from prepare_fold)
if self._geometry_source == "unet":
self._prepare_fold_unet(
list(all_paths.keys()), context.eye_train, context.device
)
else:
self._prepare_fold_gt(list(all_paths.keys()))
# 2. Inject into samples (from augment_samples)
for samples in (
context.eye_train,
context.bilat_train,
context.bilat_val,
context.bilat_test,
):
if samples:
# This modifies the list of dicts in the context object in-place
self.augment_samples(samples)
def prepare_fold(
self,
*,
eye_train,
bilat_train,
bilat_val,
bilat_test,
image_preprocessor,
image_cache,
device,
args,
) -> None:
"""Build seg-map generator and pre-compute maps for all fold images."""
if self._manifest_path is None:
raise ValueError("GeometryTower requires manifest_path")
all_paths: dict = {}
for split in (eye_train, bilat_train, bilat_val, bilat_test):
for s in split:
for slot in ("image_1", "image_2"):
p = s.get(slot)
if p is not None:
all_paths[str(Path(p).resolve())] = None
if self._geometry_source == "unet":
self._prepare_fold_unet(list(all_paths.keys()), eye_train, device)
else:
self._prepare_fold_gt(list(all_paths.keys()))
def augment_samples(self, samples: list) -> list:
"""Inject ``seg_map_1`` / ``seg_map_2`` float32 arrays into each sample dict.
Arrays have shape (C, H, W) and are collated by the DataLoader into
(B, C, H, W) tensors delivered to ``embed_batch``.
"""
blank = np.zeros(
(self._in_channels, self._target_size, self._target_size), dtype=np.float32
)
for s in samples:
for img_slot, seg_slot in (
("image_1", "seg_map_1"),
("image_2", "seg_map_2"),
):
img_path = s.get(img_slot)
if img_path is None:
continue
key = str(Path(img_path).resolve())
s[seg_slot] = self._seg_cache.get(key, blank)
return samples
def embed_batch(
self,
batch: dict,
*,
device: torch.device,
slot: int = 1,
) -> list[torch.Tensor]:
seg = batch.get(f"seg_map_{slot}")
if seg is None or not torch.is_tensor(seg):
ref = batch.get(f"image_{slot}")
bs = ref.shape[0] if torch.is_tensor(ref) else 1
return [torch.zeros(bs, self._out_dim, device=device)]
return [self._seg_cnn.backbone(seg.float().to(device))]
# ------------------------------------------------------------------
# Internal helpers
# ------------------------------------------------------------------
def _seg_map_to_array(self, seg_map: np.ndarray) -> np.ndarray:
"""Apply crop + resize and return a (C, H, W) float32 numpy array."""
if self._crop_to_disc:
seg_map = crop_to_disc(seg_map)
return seg_map_to_tensor(seg_map, self._in_channels, self._target_size).numpy()
def _prepare_fold_gt(self, image_paths: list) -> None:
"""Pre-compute GT seg maps from manifest annotations."""
manifest_df = pd.read_csv(self._manifest_path)
manifest_df["_img_key"] = manifest_df["image_path"].apply(
lambda p: str(Path(p).resolve())
)
manifest_index = manifest_df.set_index("_img_key").to_dict("index")
print(
f"[GeometryTower] pre-computing GT seg maps for {len(image_paths)} images...",
flush=True,
)
n_ok = 0
blank = np.zeros((self._seg_target_size, self._seg_target_size), dtype=np.uint8)
for img_path in image_paths:
entry = manifest_index.get(img_path)
if entry is None:
self._seg_cache[img_path] = self._seg_map_to_array(blank)
continue
rec = SegMapRecord(
sample_id="",
image_path=Path(img_path),
annotation_disc=Path(entry["annotation_disc"]),
annotation_cup=Path(entry["annotation_cup"]),
annotation_type_disc=entry["annotation_type_disc"],
annotation_type_cup=entry["annotation_type_cup"],
patient_id=0,
eye="",
label=0,
)
try:
disc_mask, cup_mask = load_gt_masks(rec, self._seg_target_size)
self._seg_cache[img_path] = self._seg_map_to_array(
_combine_masks(disc_mask, cup_mask)
)
n_ok += 1
except Exception:
self._seg_cache[img_path] = self._seg_map_to_array(blank)
print(
f"[GeometryTower] {n_ok}/{len(image_paths)} GT seg maps computed",
flush=True,
)
def _prepare_fold_unet(self, image_paths: list, eye_train: list, device) -> None:
"""Pre-compute U-Net seg maps, with optional per-fold fine-tuning."""
from v3.classes.unet_segmenter import UNetSegmenter
from torch.utils.data import DataLoader as _DL
if self._weights_path is None:
raise ValueError("GeometryTower(source='unet') requires weights_path")
segmenter = UNetSegmenter(
manifest_path=self._manifest_path,
normalize=self._unet_normalize,
)
state = torch.load(self._weights_path, map_location=segmenter.device)
segmenter.model.load_state_dict(state.get("model", state))
segmenter.model.to(segmenter.device).eval()
if self._finetune_unet_epochs > 0:
print(
f"[GeometryTower] fine-tuning U-Net for {self._finetune_unet_epochs} epochs...",
flush=True,
)
ft_loader = _DL(
UNetFineTuneDataset(
self._build_records_from_samples(eye_train),
target_size=segmenter.target_size,
normalize=self._unet_normalize,
),
batch_size=4,
shuffle=True,
num_workers=0,
)
optimizer = torch.optim.Adam(
segmenter.model.parameters(), lr=self._finetune_unet_lr
)
criterion = torch.nn.BCEWithLogitsLoss()
segmenter.model.train()
for _ in range(self._finetune_unet_epochs):
for images, masks in ft_loader:
images, masks = images.to(segmenter.device), masks.to(
segmenter.device
)
optimizer.zero_grad()
criterion(segmenter.model(images), masks).backward()
optimizer.step()
segmenter.model.eval()
print(
f"[GeometryTower] running U-Net inference on {len(image_paths)} images...",
flush=True,
)
records = [
SegMapRecord(
sample_id="",
image_path=Path(p),
annotation_disc=Path(p),
annotation_cup=Path(p),
annotation_type_disc="",
annotation_type_cup="",
patient_id=0,
eye="",
label=0,
)
for p in image_paths
]
seg_maps = precompute_unet_seg_maps(records, segmenter, self._unet_threshold)
for img_path, seg_map in zip(image_paths, seg_maps):
self._seg_cache[img_path] = self._seg_map_to_array(seg_map)
print(f"[GeometryTower] {len(seg_maps)} U-Net seg maps cached", flush=True)
def _build_records_from_samples(self, samples: list) -> list:
"""Build SegMapRecord list from HyperTower sample dicts (for U-Net fine-tuning)."""
manifest_df = pd.read_csv(self._manifest_path)
manifest_df["_img_key"] = manifest_df["image_path"].apply(
lambda p: str(Path(p).resolve())
)
manifest_index = manifest_df.set_index("_img_key").to_dict("index")
records = []
for s in samples:
for slot in ("image_1", "image_2"):
p = s.get(slot)
if p is None:
continue
key = str(Path(p).resolve())
entry = manifest_index.get(key)
if entry is None:
continue
records.append(
SegMapRecord(
sample_id="",
image_path=Path(p),
annotation_disc=Path(entry["annotation_disc"]),
annotation_cup=Path(entry["annotation_cup"]),
annotation_type_disc=entry["annotation_type_disc"],
annotation_type_cup=entry["annotation_type_cup"],
patient_id=int(s.get("patient_id", 0)),
eye=str(s.get("eye", "")),
label=int(s.get("label", 0)),
)
)
return records
v3/classes/hypertower_logger.py
+128
View File
@@ -0,0 +1,128 @@
from __future__ import annotations
import csv
import json
import logging
from pathlib import Path
from typing import Optional
DEFAULT_OPTIONAL_EPOCH_COLS = [
"pct_fused",
"pct_img",
"pct_md",
"phase",
"se_mean",
"se_std",
"se_pct_lt_0.2",
"se_pct_gt_0.8",
"holdout_loss",
"holdout_acc_fused",
"holdout_acc_img",
"holdout_acc_cd",
"holdout_auc_fused",
"holdout_auc_img",
"holdout_auc_cd",
"best_monitor",
"best_so_far",
"best_epoch",
"early_best_so_far",
"early_bad_epochs",
"early_improved",
"early_monitor",
"holdout_best_monitor",
"holdout_best_so_far",
"holdout_best_epoch",
]
class HypertowerLogger:
"""
Shared logging utility for V2 tower workflows.
- train.log line logging
- epoch_log.csv row logging with stable header
- lightweight JSON/array artifact helpers
"""
def __init__(
self,
*,
run_dir: Path,
train_log_path: Optional[Path] = None,
epoch_log_path: Optional[Path] = None,
logger_name: Optional[str] = None,
) -> None:
self.run_dir = Path(run_dir).resolve()
self.run_dir.mkdir(parents=True, exist_ok=True)
self.train_log_path = Path(train_log_path) if train_log_path else (self.run_dir / "train.log")
self.epoch_log_path = Path(epoch_log_path) if epoch_log_path else (self.run_dir / "epoch_log.csv")
self._logger_name = logger_name or f"hypertower.{id(self)}"
self.logger = logging.getLogger(self._logger_name)
self.logger.setLevel(logging.INFO)
self.logger.handlers = []
fh = logging.FileHandler(str(self.train_log_path))
fh.setFormatter(logging.Formatter("%(asctime)s - %(message)s"))
self.logger.addHandler(fh)
self.logger.propagate = False
self._epoch_log_fp = None
self._epoch_log_writer = None
self._epoch_log_fields: list[str] | None = None
def info(self, msg: str) -> None:
self.logger.info(msg)
def warning(self, msg: str) -> None:
self.logger.warning(msg)
def error(self, msg: str) -> None:
self.logger.error(msg)
def write_epoch_row(
self,
row: dict,
*,
path: str | Path | None = None,
optional_cols: Optional[list[str]] = None,
) -> None:
optional = optional_cols if optional_cols is not None else DEFAULT_OPTIONAL_EPOCH_COLS
if self._epoch_log_writer is None:
fieldnames = list(dict.fromkeys([*row.keys(), *optional]))
target_path = Path(path) if path is not None else self.epoch_log_path
target_path.parent.mkdir(parents=True, exist_ok=True)
self._epoch_log_fp = open(target_path, "w", newline="", encoding="utf-8")
self._epoch_log_writer = csv.DictWriter(self._epoch_log_fp, fieldnames=fieldnames)
self._epoch_log_writer.writeheader()
self._epoch_log_fields = fieldnames
assert self._epoch_log_fields is not None
assert self._epoch_log_writer is not None
assert self._epoch_log_fp is not None
for key in self._epoch_log_fields:
row.setdefault(key, None)
self._epoch_log_writer.writerow({k: row.get(k) for k in self._epoch_log_fields})
self._epoch_log_fp.flush()
def write_json(self, path: str | Path, payload: dict) -> None:
target = Path(path)
if not target.is_absolute():
target = self.run_dir / target
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(json.dumps(payload, indent=2), encoding="utf-8")
def close(self) -> None:
if self._epoch_log_fp is not None:
try:
self._epoch_log_fp.close()
except Exception:
pass
self._epoch_log_fp = None
self._epoch_log_writer = None
self._epoch_log_fields = None
for handler in list(self.logger.handlers):
try:
handler.close()
except Exception:
pass
self.logger.removeHandler(handler)

Some files were not shown because too many files have changed in this diff Show More