diff --git a/.gitignore b/.gitignore index 98d783a..dccb51a 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ models/v2/refuge/ .archive/ scripts/deprecated/ v3/results/* +scripts/utility/backup_mirror_with_archive.sh +v3/distributed/logs/* \ No newline at end of file diff --git a/scripts/utility/backup_mirror_with_archive.sh b/scripts/utility/backup_mirror_with_archive.sh index 3c3ebb9..d5d09d4 100755 --- a/scripts/utility/backup_mirror_with_archive.sh +++ b/scripts/utility/backup_mirror_with_archive.sh @@ -44,7 +44,42 @@ cmd=( --delete --backup --backup-dir="$archive_dir" + # Directories with no backup value + --exclude=".git/" + --exclude=".claude/" --exclude=".archive/" + # Large datasets stored elsewhere + --exclude="refuge/" + --exclude="Refuge/" + --exclude="REFUGE/" + --exclude="papila/" + --exclude="Papila/" + --exclude="PAPILA/" + # Python / general caches + --exclude="__pycache__/" + --exclude=".mypy_cache/" + --exclude=".ruff_cache/" + --exclude=".pytest_cache/" + --exclude=".cache/" + --exclude="*.pyc" + --exclude="*.pyo" + # Virtual environments + --exclude=".venv/" + --exclude="venv/" + --exclude="env/" + # Node + --exclude="node_modules/" + # Build / dist artifacts + --exclude="*.egg-info/" + --exclude="dist/" + --exclude="build/" + --exclude=".eggs/" + # IDE / editor metadata + --exclude=".idea/" + --exclude=".vscode/" + # OS metadata + --exclude=".DS_Store" + --exclude="Thumbs.db" "${SOURCE%/}/" "${DEST%/}/" ) diff --git a/setup_env.sh b/setup_env.sh new file mode 100755 index 0000000..4f1d824 --- /dev/null +++ b/setup_env.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# ============================================================ +# HyperTower environment setup script. +# +# Installs all required packages for the v3 pipeline. +# PyTorch is installed CPU-only by default. +# +# To upgrade to GPU after running this script: +# NVIDIA (CUDA 12.8): +# pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 +# AMD (ROCm 6.2): +# pip install torch torchvision --index-url https://download.pytorch.org/whl/rocm6.2 +# +# Usage: +# bash setup_env.sh +# ============================================================ + +set -e + +echo "=== HyperTower environment setup ===" +echo "Installing CPU-only PyTorch (upgrade separately for GPU)" +echo "" + +# ── PyTorch (CPU) ───────────────────────────────────────────── +pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu + +# ── Core data science ───────────────────────────────────────── +pip install \ + numpy \ + pandas \ + scikit-learn \ + scipy + +# ── Image processing ────────────────────────────────────────── +pip install \ + Pillow \ + scikit-image + +# ── Visualisation ───────────────────────────────────────────── +pip install \ + matplotlib + +# ── Distributed job system ──────────────────────────────────── +pip install \ + fastapi \ + "uvicorn[standard]" \ + requests \ + pydantic + +# ── Statistics ──────────────────────────────────────────────── +pip install \ + statsmodels + +# ── Utilities ───────────────────────────────────────────────── +pip install \ + tqdm \ + openpyxl + +echo "" +echo "=== Setup complete ===" +echo "" +echo "To enable GPU support:" +echo " NVIDIA: pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128" +echo " AMD: pip install torch torchvision --index-url https://download.pytorch.org/whl/rocm6.2" diff --git a/v3/classes/bridges.py b/v3/classes/bridges.py index b2783bb..dd0a8de 100644 --- a/v3/classes/bridges.py +++ b/v3/classes/bridges.py @@ -14,6 +14,7 @@ class Bridge(nn.Module): num_classes, fusion_dim=256, mode="fused", + dropout: float = 0.5, use_se: bool = True, se_reduction: int = 16, se_pre_norm: bool = True, @@ -37,7 +38,7 @@ class Bridge(nn.Module): # heads self.classifier_fused = nn.Sequential( nn.ReLU(), - nn.Dropout(0.5), + nn.Dropout(dropout), nn.Linear(fusion_dim, num_classes), ) self.classifier_img = nn.Linear(img_dim, num_classes) @@ -54,26 +55,28 @@ class Bridge(nn.Module): return self.se_log.get(reset=reset) return None + def _compute_fused(self, img_feats, md_feats): + """Return z_fused embedding (before classifier_fused). Used by encode() and forward().""" + hi = self.ln_img(self.W_img(img_feats)) + hm = self.ln_md(self.W_md(md_feats)) + fused = hi * hm + if self.se is not None: + fused, gates = self.se(fused) + if self.se_log.enabled: + self.se_log.accumulate(gates) + return fused + + def encode(self, img_feats, md_feats) -> torch.Tensor: + """Return z_fused embedding without applying the classifier head.""" + assert self.mode == "fused", "encode() only valid in fused mode" + return self._compute_fused(img_feats, md_feats) + def forward(self, img_feats, md_feats): out_img = None if self.mode == "clinical_only" else self.classifier_img(img_feats) out_md = None if self.mode == "image_only" else self.classifier_cd(md_feats) if self.mode == "fused": - hi = self.ln_img(self.W_img(img_feats)) # image features - hm = self.ln_md(self.W_md(md_feats)) # clinical data features - fused = hi * hm # elementwise product - # apply SE gates - if self.se is not None: - fused, gates = self.se(fused) - if self.se_log.enabled: - self.se_log.accumulate(gates) - - if self.se is not None and self.training and self.se_log.enabled: - if not hasattr(self, "_dbg_seen"): - self._dbg_seen = 0 - if self._dbg_seen < 3: # print only a few times - print("[SE] gate mean this batch:", gates.mean().item()) - self._dbg_seen += 1 + fused = self._compute_fused(img_feats, md_feats) out_f = self.classifier_fused(fused) return out_f, out_img, out_md # if ablation modes: diff --git a/v3/classes/croppers.py b/v3/classes/croppers.py index fe91273..adcef50 100644 --- a/v3/classes/croppers.py +++ b/v3/classes/croppers.py @@ -189,6 +189,31 @@ class UNetImageCropper: return None return np.asarray(features, dtype=np.float32) + def precompute_geometry(self, image_paths) -> None: + """Pre-compute geometry features for all image_paths into an in-memory cache. + Safe to call in the main process; geometry_for_image() can then be called + from DataLoader workers without touching CUDA. + """ + self._geometry_cache: Dict[str, Optional[np.ndarray]] = {} + paths = list(image_paths) + print(f"[UNetImageCropper] pre-computing geometry for {len(paths)} images...", flush=True) + for img_path in paths: + key = str(Path(img_path).resolve()) + try: + img = Image.open(img_path).convert("RGB") + self._geometry_cache[key] = self.geometry_features(img, img_path) + except Exception: + self._geometry_cache[key] = None + n_ok = sum(1 for v in self._geometry_cache.values() if v is not None) + print(f"[UNetImageCropper] {n_ok}/{len(paths)} geometry vectors computed", flush=True) + + def geometry_for_image(self, image_path) -> Optional[np.ndarray]: + """Return pre-computed geometry vector for image_path (call precompute_geometry first).""" + cache = getattr(self, "_geometry_cache", None) + if cache is None: + raise RuntimeError("Call precompute_geometry() before geometry_for_image()") + return cache.get(str(Path(image_path).resolve())) + class ManifestImageCropper: def __init__( @@ -371,6 +396,35 @@ class ManifestImageCropper: return None return np.asarray(features, dtype=np.float32) + def precompute_geometry(self, image_paths) -> None: + """Pre-compute geometry features for all image_paths into an in-memory cache. + Safe to call in the main process; geometry_for_image() can then be called + without re-opening images or re-loading annotations. + """ + self._geometry_cache: Dict[str, Optional[np.ndarray]] = {} + paths = list(image_paths) + print(f"[ManifestImageCropper] pre-computing geometry for {len(paths)} images...", flush=True) + for img_path in paths: + key = str(Path(img_path).resolve()) + entry = self.entries.get(key) + if entry is None: + self._geometry_cache[key] = None + continue + try: + img = Image.open(img_path).convert("RGB") + self._geometry_cache[key] = self.geometry_features(img, img_path) + except Exception: + self._geometry_cache[key] = None + n_ok = sum(1 for v in self._geometry_cache.values() if v is not None) + print(f"[ManifestImageCropper] {n_ok}/{len(paths)} geometry vectors computed", flush=True) + + def geometry_for_image(self, image_path) -> Optional[np.ndarray]: + """Return pre-computed geometry vector for image_path (call precompute_geometry first).""" + cache = getattr(self, "_geometry_cache", None) + if cache is None: + raise RuntimeError("Call precompute_geometry() before geometry_for_image()") + return cache.get(str(Path(image_path).resolve())) + # --------------------------------------------------------------------------- # Factory diff --git a/v3/classes/loader_factory.py b/v3/classes/loader_factory.py index f27d940..6174a3e 100644 --- a/v3/classes/loader_factory.py +++ b/v3/classes/loader_factory.py @@ -211,6 +211,7 @@ def make_loader( shuffle: bool, num_workers: int, sampler: Optional[WeightedRandomSampler] = None, + persistent_workers: bool = False, ) -> DataLoader: ds = SlotDataset( samples, @@ -226,6 +227,7 @@ def make_loader( sampler=sampler, num_workers=num_workers, collate_fn=slot_collate, + persistent_workers=(persistent_workers and num_workers > 0), ) diff --git a/v3/classes/models.py b/v3/classes/models.py index f581f45..5fd5070 100644 --- a/v3/classes/models.py +++ b/v3/classes/models.py @@ -11,7 +11,7 @@ from torch import nn from torch.utils.data import DataLoader from v3.classes.bridges import Bridge -from v3.classes.towers import ImageTower, ClinicalTower +from v3.classes.towers import ImageTower, ClinicalTower, SiameseImageTower # --------------------------------------------------------------------------- @@ -35,18 +35,24 @@ class SingleEyeHT(nn.Module): cd_hidden_dim: int = 128, fusion_dim: int = 256, bridge_mode: str = "fused", + bridge_dropout: float = 0.5, + cd_dropout: float = 0.1, + se_img_tower: bool = False, + se_cd_tower: bool = False, + se_bridge: bool = False, ): super().__init__() self.img_tower = ImageTower( backbone=backbone, freeze_ratio=freeze_ratio, augment=augment, - use_se=False, + use_se=se_img_tower, ) self.cd_tower = ClinicalTower( clinical_data=clinical_data, hidden_dim=cd_hidden_dim, - use_se=False, + dropout=cd_dropout, + use_se=se_cd_tower, ) self.bridge = Bridge( img_dim=self.img_tower.out_dim, @@ -54,13 +60,20 @@ class SingleEyeHT(nn.Module): num_classes=num_classes, fusion_dim=fusion_dim, mode=bridge_mode, - use_se=False, + dropout=bridge_dropout, + use_se=se_bridge, ) @property def transform(self): return self.img_tower.transform + def encode(self, x: torch.Tensor, meta: torch.Tensor) -> torch.Tensor: + """Return z_fused embedding (fusion_dim) without applying the classifier head.""" + img_feats = self.img_tower(x) + md_feats = self.cd_tower(meta) + return self.bridge.encode(img_feats, md_feats) + def forward(self, x: torch.Tensor, meta: torch.Tensor) -> torch.Tensor: img_feats = None if self.bridge.mode == "clinical_only" else self.img_tower(x) md_feats = None if self.bridge.mode == "image_only" else self.cd_tower(meta) @@ -158,6 +171,59 @@ class BilateralHT(nn.Module): return out_f +class SiameseHT(nn.Module): + """ + Bilateral model using a shared-weight SiameseImageTower (mean+delta). + + Both eyes pass through the same backbone; features are combined as + cat([mean(f_od, f_os), f_od - f_os]) giving the model both a shared + bilateral representation and an asymmetry signal. + + Uses the same bilateral loader and training loop as BilateralHT. + The bridge operates in image_only mode (no clinical data in phase 4). + """ + + def __init__( + self, + *, + backbone: str, + freeze_ratio: float, + augment: bool, + num_classes: int, + fusion_dim: int = 256, + bridge_mode: str = "image_only", + ): + super().__init__() + self.img_tower = SiameseImageTower( + backbone=backbone, + freeze_ratio=freeze_ratio, + augment=augment, + use_se=False, + ) + img_dim = self.img_tower.out_dim # 2 * backbone_out_dim + self.bridge = Bridge( + img_dim=img_dim, + meta_dim=1, # dummy — not used in image_only mode + num_classes=num_classes, + fusion_dim=fusion_dim, + mode="image_only", + use_se=False, + ) + self.aux_img = nn.Linear(img_dim, num_classes) + + @property + def transform(self): + return self.img_tower.transform + + def encode(self, x_od: torch.Tensor, x_os: torch.Tensor) -> torch.Tensor: + return self.img_tower(x_od, x_os) + + def forward(self, x_od: torch.Tensor, x_os: torch.Tensor) -> torch.Tensor: + feats = self.encode(x_od, x_os) + out_f, _, _ = self.bridge(feats, None) + return out_f + + class FusedEnsembleHT(nn.Module): """ SingleEyeHT base with a per-eye attention scorer for bilateral fusion. @@ -190,6 +256,10 @@ class FusedEnsembleHT(nn.Module): # Learns the GC-direction in logit space from bilateral labels. self.eye_scorer = nn.Linear(num_classes, 1, bias=True) + @property + def head(self) -> nn.Module: + return self.eye_scorer + def forward( self, x_od: torch.Tensor, @@ -205,6 +275,78 @@ class FusedEnsembleHT(nn.Module): return alpha[:, 0:1] * logit_od + alpha[:, 1:2] * logit_os # [B, C] +class LogitMLPEnsembleHT(nn.Module): + """ + MLP head trained on concatenated per-eye logits. + + Both eyes pass through the frozen base independently, producing per-eye + logit vectors. These are concatenated and fed through a small MLP: + + cat([logit_od, logit_os]) [B, 2C] + → Linear(2C, hidden) → ReLU → Dropout → Linear(hidden, C) + + Permutation-variant by design: the model can learn left/right asymmetries + directly from the concatenated pair, at the cost of needing a consistent + OD-first input ordering. + """ + + def __init__(self, base: SingleEyeHT, num_classes: int, hidden: int = 64): + super().__init__() + self.base = base + self.head = nn.Sequential( + nn.Linear(2 * num_classes, hidden), + nn.ReLU(), + nn.Dropout(0.3), + nn.Linear(hidden, num_classes), + ) + + def forward( + self, + x_od: torch.Tensor, meta_od: torch.Tensor, + x_os: torch.Tensor, meta_os: torch.Tensor, + ) -> torch.Tensor: + logit_od = self.base(x_od, meta_od) + logit_os = self.base(x_os, meta_os) + return self.head(torch.cat([logit_od, logit_os], dim=1)) + + +class EmbeddingMLPEnsembleHT(nn.Module): + """ + MLP head trained on concatenated per-eye z_fused embeddings. + + Both eyes pass through the frozen base independently, and their bridge + embeddings (pre-classifier, shape [B, fusion_dim]) are concatenated and + fed through an MLP: + + cat([z_od, z_os]) [B, 2 * fusion_dim] + → Linear(2*fusion_dim, hidden) → ReLU → Dropout → Linear(hidden, C) + + Richer than logit-level: the head sees pre-softmax feature vectors rather + than the compressed C-dimensional output, giving it more signal to work + with when fusion_dim >> C. + """ + + def __init__(self, base: SingleEyeHT, num_classes: int, hidden: int = 256): + super().__init__() + self.base = base + fusion_dim = base.bridge.W_img.out_features + self.head = nn.Sequential( + nn.Linear(2 * fusion_dim, hidden), + nn.ReLU(), + nn.Dropout(0.3), + nn.Linear(hidden, num_classes), + ) + + def forward( + self, + x_od: torch.Tensor, meta_od: torch.Tensor, + x_os: torch.Tensor, meta_os: torch.Tensor, + ) -> torch.Tensor: + z_od = self.base.encode(x_od, meta_od) + z_os = self.base.encode(x_os, meta_os) + return self.head(torch.cat([z_od, z_os], dim=1)) + + # --------------------------------------------------------------------------- # Phase control # --------------------------------------------------------------------------- @@ -429,15 +571,89 @@ def train_bilateral_epoch( ) +def train_siamese_epoch( + model: SiameseHT, + loader: DataLoader, + opt, + device: torch.device, + *, + bcd_prob: float = 0.5, + tower_loss_mode: str = "bcd", +) -> tuple[float, float]: + """Train one epoch of SiameseHT on bilateral (patient-level) samples.""" + model.train() + total_loss = total_correct = total_n = 0 + for batch in loader: + x1 = batch.get("image_1") + x2 = batch.get("image_2") + y = batch.get("label_1") + if not (torch.is_tensor(x1) and torch.is_tensor(x2)): + continue + x1 = x1.to(device); x2 = x2.to(device) + y = _to_label_tensor(y, device) + feats = model.encode(x1, x2) + + if tower_loss_mode == "all": + logits_i = model.aux_img(feats) + out_f, _, _ = model.bridge(feats, None) + loss = F.cross_entropy(out_f, y) + F.cross_entropy(logits_i, y) + logits = out_f + elif random() < bcd_prob: + logits = model.aux_img(feats) + loss = F.cross_entropy(logits, y) + else: + logits, _, _ = model.bridge(feats, None) + loss = F.cross_entropy(logits, y) + + opt.zero_grad() + loss.backward() + opt.step() + bs = y.shape[0] + total_loss += float(loss.item()) * bs + total_correct += int((logits.argmax(1) == y).sum()) + total_n += bs + return ( + total_loss / total_n if total_n else float("nan"), + total_correct / total_n if total_n else float("nan"), + ) + + +def collect_probs_siamese( + model: SiameseHT, + loader: DataLoader, + device: torch.device, +) -> tuple[np.ndarray, np.ndarray]: + """Patient-level probs from a bilateral loader using SiameseHT.""" + model.eval() + y_c, p_c = [], [] + with torch.no_grad(): + for batch in loader: + x1 = batch.get("image_1") + x2 = batch.get("image_2") + y = batch.get("label_1") + if not (torch.is_tensor(x1) and torch.is_tensor(x2)): + continue + x1 = x1.to(device); x2 = x2.to(device) + feats = model.encode(x1, x2) + logits, _, _ = model.bridge(feats, None) + probs = torch.softmax(logits, dim=1) + y_c.append(np.array(y) if not torch.is_tensor(y) else y.cpu().numpy()) + p_c.append(probs.cpu().numpy()) + return ( + np.concatenate(y_c, axis=0), + np.concatenate(p_c, axis=0), + ) + + def train_fusion_epoch( - model: FusedEnsembleHT, + model, # FusedEnsembleHT | LogitMLPEnsembleHT | EmbeddingMLPEnsembleHT loader: DataLoader, opt, device: torch.device, ) -> tuple[float, float]: """Train only the fusion head; the base SingleEyeHT is frozen in eval mode.""" model.base.eval() - model.eye_scorer.train() + model.head.train() total_loss = total_correct = total_n = 0 for batch in loader: x1 = batch.get("image_1"); m1 = batch.get("matrix_1") diff --git a/v3/classes/predictions.py b/v3/classes/predictions.py index 009ec39..5fe6d51 100644 --- a/v3/classes/predictions.py +++ b/v3/classes/predictions.py @@ -31,6 +31,8 @@ def head_names_for_mode(tower_mode: str, *, fused_head: bool = False) -> list[st return names + ["bilat_fused"] if fused_head else names if tower_mode == "bilateral": return ["fused", "img_joint", "md_joint"] + if tower_mode == "siamese": + return ["fused"] raise ValueError(f"Unknown tower_mode: {tower_mode!r}") diff --git a/v3/classes/seg_cnn.py b/v3/classes/seg_cnn.py new file mode 100644 index 0000000..6ea3a02 --- /dev/null +++ b/v3/classes/seg_cnn.py @@ -0,0 +1,521 @@ +"""Segmentation-map CNN for glaucoma grading. + +Trains a CNN on combined disc/cup segmentation maps — pixel values + 0 = background, 1 = disc (rim only), 2 = cup +— instead of raw RGB fundus images, forcing the model to learn purely +from optic nerve head geometry (CDR, rim width, cup location, etc.). + +Two segmentation sources are supported: + gt – rasterise expert contour/mask annotations directly (pure NumPy/PIL, + no CUDA — safe in DataLoader worker processes) + unet – run a trained UNetSegmenter on the raw fundus image + +Usage (import from training script): + from v3.classes.seg_cnn import SegMapRecord, SegMapDataset, SegCNN, seg_map_to_tensor +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Tuple + +import numpy as np +import torch +import torch.nn as nn +from PIL import Image, ImageDraw +from PIL.Image import Resampling +from torch.utils.data import Dataset +from torchvision import models, transforms +from tqdm import tqdm + + +# --------------------------------------------------------------------------- +# Data record +# --------------------------------------------------------------------------- + +@dataclass +class SegMapRecord: + """One labelled eye sample for the seg-map CNN.""" + sample_id: str + image_path: Path # original fundus image (used by unet mode) + annotation_disc: Path # contour (.txt) or mask (.bmp/.png) + annotation_cup: Path + annotation_type_disc: str # "contour" or "mask" + annotation_type_cup: str + patient_id: int # for group-CV: keep both eyes of a patient together + eye: str # "OD" or "OS" + label: int # 0 = Normal, 1 = Glaucoma + + +# --------------------------------------------------------------------------- +# Seg-map utilities +# --------------------------------------------------------------------------- + +def _combine_masks(disc_mask: np.ndarray, cup_mask: np.ndarray) -> np.ndarray: + """ + Combine binary disc and cup masks into a 3-class label map. + + Returns a uint8 array with values: + 0 — background + 1 — optic disc rim (disc but not cup) + 2 — optic cup + """ + disc = (disc_mask > 0).astype(np.uint8) + cup = (cup_mask > 0).astype(np.uint8) + cup = (cup & disc) # structural prior: cup must be inside disc + seg = disc + cup # 0, 1 (rim), or 2 (cup) + return seg.astype(np.uint8) + + +def crop_to_disc(seg_map: np.ndarray) -> np.ndarray: + """ + Crop a seg map tightly to the disc bounding box. + + The disc is anywhere seg_map > 0 (i.e. rim or cup). + Returns the original array unchanged if no disc is found. + """ + rows = np.any(seg_map > 0, axis=1) + cols = np.any(seg_map > 0, axis=0) + if not rows.any(): + return seg_map + r0, r1 = int(np.argmax(rows)), int(len(rows) - 1 - np.argmax(rows[::-1])) + c0, c1 = int(np.argmax(cols)), int(len(cols) - 1 - np.argmax(cols[::-1])) + return seg_map[r0:r1 + 1, c0:c1 + 1] + + +def seg_map_to_tensor( + seg_map: np.ndarray, + channels: int, + target_size: int, +) -> torch.Tensor: + """ + Convert an (H, W) seg map with values {0, 1, 2} to a float tensor. + + channels=1 → (1, H, W) float in [0, 1] (values 0/0.5/1.0) + channels=3 → (3, H, W) one-hot binary channels [bg, disc_rim, cup] + """ + pil = Image.fromarray(seg_map.astype(np.uint8), mode="L") + pil = pil.resize((target_size, target_size), Image.NEAREST) + seg = np.array(pil, dtype=np.uint8) + + if channels == 1: + arr = seg.astype(np.float32) / 2.0 # {0, 0.5, 1.0} + return torch.from_numpy(arr).unsqueeze(0) + + if channels == 3: + bg = (seg == 0).astype(np.float32) + disc_rim = (seg == 1).astype(np.float32) + cup = (seg == 2).astype(np.float32) + return torch.from_numpy(np.stack([bg, disc_rim, cup], axis=0)) + + raise ValueError(f"channels must be 1 or 3, got {channels}") + + +# --------------------------------------------------------------------------- +# GT mask loading (pure NumPy / PIL — no CUDA, safe in DataLoader workers) +# --------------------------------------------------------------------------- + +def _load_contour(path: Path) -> np.ndarray: + """Load x,y contour pairs from a whitespace- or comma-delimited text file.""" + for delimiter in (",", None): + try: + arr = np.loadtxt(str(path), delimiter=delimiter, comments="#", dtype=np.float32) + if arr.size > 0: + break + except Exception: + arr = np.zeros((0, 2), dtype=np.float32) + if arr.size == 0 or arr.ndim == 1: + return np.zeros((0, 2), dtype=np.float32) + if arr.shape[1] < 2: + return np.zeros((0, 2), dtype=np.float32) + return arr[:, :2] + + +def _contour_to_mask( + coords: np.ndarray, image_size: Tuple[int, int], target_size: int +) -> np.ndarray: + """ + Rasterise a polygon defined by (x, y) coords into a binary mask. + + image_size is the (width, height) of the original fundus image — the + coordinate space the contour was annotated in. The mask is drawn at + that resolution then resized to target_size, matching UNetSegmenter's + behaviour and avoiding off-canvas clipping. + """ + if coords is None or len(coords) < 3: + return np.zeros((target_size, target_size), dtype=np.uint8) + points = [tuple(map(float, pt)) for pt in coords] + img = Image.new("L", image_size, 0) + ImageDraw.Draw(img).polygon(points, outline=1, fill=1) + img = img.resize((target_size, target_size), Resampling.NEAREST) + return (np.array(img, dtype=np.uint8) > 0).astype(np.uint8) + + +def _extract_masks_from_image( + mask_path: Path, target_size: int +) -> Tuple[np.ndarray, np.ndarray]: + """ + Extract disc and cup binary masks from a segmentation image file. + + Handles both grayscale label images (e.g. REFUGE .bmp) and + RGB colour-coded masks. Returns (disc_mask, cup_mask) both at + target_size × target_size. + """ + raw = Image.open(mask_path) + arr = np.array(raw) + + if arr.ndim == 2: + # Grayscale: identify background from edge statistics + edges = np.concatenate([arr[0], arr[-1], arr[:, 0], arr[:, -1]]) + bg_val = int(np.argmax(np.bincount(edges.astype(np.int64).clip(0, 255), minlength=256))) + disc_arr = (arr != bg_val).astype(np.uint8) + vals = np.unique(arr) + non_bg = vals[vals != bg_val] + cup_arr: np.ndarray + if non_bg.size > 1: + cup_val = int(non_bg.min()) + cup_arr = (arr == cup_val).astype(np.uint8) + else: + cup_arr = np.zeros_like(disc_arr, dtype=np.uint8) + else: + img_rgb = raw.convert("RGB") + arr = np.array(img_rgb) + h, w, c = arr.shape + edges_rgb = np.concatenate( + [arr[0], arr[-1], arr[:, 0], arr[:, -1]], axis=0 + ) + edge_colors, edge_counts = np.unique(edges_rgb.reshape(-1, c), axis=0, return_counts=True) + bg_color = edge_colors[int(np.argmax(edge_counts))] + colors, counts = np.unique(arr.reshape(-1, c), axis=0, return_counts=True) + not_bg = np.any(colors != bg_color.reshape(1, -1), axis=1) + colors, counts = colors[not_bg], counts[not_bg] + disc_arr = np.zeros((h, w), dtype=np.uint8) + cup_arr = np.zeros((h, w), dtype=np.uint8) + if colors.shape[0] >= 1: + order = np.argsort(-counts) + disc_color = colors[order[0]] + disc_arr[np.all(arr == disc_color, axis=-1)] = 1 + if colors.shape[0] >= 2: + cup_color = colors[order[1]] + cup_arr[np.all(arr == cup_color, axis=-1)] = 1 + + # Resize to target_size with nearest-neighbour to preserve binary values + def _resize(m: np.ndarray) -> np.ndarray: + pil = Image.fromarray((m > 0).astype(np.uint8) * 255) + pil = pil.resize((target_size, target_size), Resampling.NEAREST) + return (np.array(pil) > 0).astype(np.uint8) + + return _resize(disc_arr), _resize(cup_arr) + + +def load_gt_masks(rec: "SegMapRecord", target_size: int) -> Tuple[np.ndarray, np.ndarray]: + """ + Load GT disc + cup masks for one record. + + Handles annotation_type "contour" (x,y text file) and "mask" (image file). + Returns (disc_mask, cup_mask) as uint8 arrays of shape (target_size, target_size). + """ + disc_mask: Optional[np.ndarray] = None + cup_mask: Optional[np.ndarray] = None + + # Get original image size so contour coordinates are drawn in the right space + with Image.open(rec.image_path) as _img: + image_size = _img.size # (width, height) + + # ---- Disc ---- + if rec.annotation_type_disc == "mask": + disc_mask, cup_from_disc = _extract_masks_from_image(rec.annotation_disc, target_size) + if cup_from_disc.any(): + cup_mask = cup_from_disc + else: # contour + coords = _load_contour(rec.annotation_disc) + disc_mask = _contour_to_mask(coords, image_size, target_size) + + # ---- Cup ---- + if cup_mask is None: + if rec.annotation_type_cup == "mask": + _, cup_from_cup = _extract_masks_from_image(rec.annotation_cup, target_size) + cup_mask = cup_from_cup + else: # contour + coords = _load_contour(rec.annotation_cup) + cup_mask = _contour_to_mask(coords, image_size, target_size) + + if disc_mask is None: + disc_mask = np.zeros((target_size, target_size), dtype=np.uint8) + if cup_mask is None: + cup_mask = np.zeros((target_size, target_size), dtype=np.uint8) + + # Structural prior: cup must lie within disc + cup_mask = (cup_mask > 0) & (disc_mask > 0) + return disc_mask.astype(np.uint8), cup_mask.astype(np.uint8) + + +# --------------------------------------------------------------------------- +# U-Net fine-tuning dataset +# --------------------------------------------------------------------------- + +class UNetFineTuneDataset(Dataset): + """ + Loads (image_tensor, mask_tensor) pairs for fine-tuning the U-Net on + PAPILA GT annotations. Uses the same preprocessing as UNetSegmenter + so the fine-tuned weights are compatible with inference. + """ + + def __init__( + self, + records: List[SegMapRecord], + target_size: int = 512, + normalize: str = "per_image", + ) -> None: + self.records = records + self.target_size = target_size + self.normalize = normalize + self.to_tensor = transforms.ToTensor() + + def __len__(self) -> int: + return len(self.records) + + def _normalize(self, tensor: torch.Tensor) -> torch.Tensor: + if self.normalize == "per_image": + mean = tensor.mean(dim=(1, 2), keepdim=True) + std = tensor.std(dim=(1, 2), keepdim=True).clamp(min=1e-6) + return (tensor - mean) / std + if self.normalize == "imagenet": + mean = torch.tensor([0.485, 0.456, 0.406]).view(-1, 1, 1) + std = torch.tensor([0.229, 0.224, 0.225]).view(-1, 1, 1) + return (tensor - mean) / std + return tensor + + def __getitem__(self, idx: int): + rec = self.records[idx] + image = Image.open(rec.image_path).convert("RGB") + image = image.resize((self.target_size, self.target_size), Resampling.BILINEAR) + img_tensor = self._normalize(self.to_tensor(image)) + + disc_mask, cup_mask = load_gt_masks(rec, self.target_size) + mask_tensor = torch.from_numpy( + np.stack([disc_mask, cup_mask], axis=0).astype(np.float32) + ) + return img_tensor, mask_tensor + + +# --------------------------------------------------------------------------- +# U-Net precomputation (run once per full record list, not per fold) +# --------------------------------------------------------------------------- + +def precompute_unet_seg_maps( + records: List["SegMapRecord"], + segmenter, + threshold: float = 0.5, +) -> List[np.ndarray]: + """ + Run the U-Net on every record and return a list of combined seg maps. + + Call this once before the CV loop and pass the results to each fold's + SegMapDataset via precomputed_seg_maps, so the U-Net isn't re-run per fold. + """ + to_tensor = transforms.ToTensor() + seg_maps = [] + for rec in tqdm(records, desc="U-Net inference", unit="img", leave=False): + image = Image.open(rec.image_path).convert("RGB") + resized = segmenter.preprocess_image(image) + tensor = segmenter._normalize_tensor( + to_tensor(resized).to(segmenter.device) + ).unsqueeze(0) + with torch.no_grad(): + logits = segmenter.model(tensor) + probs = torch.sigmoid(logits)[0].cpu().numpy() + disc = (probs[0] > threshold).astype(np.uint8) + cup = (probs[1] > threshold).astype(np.uint8) + cup = (cup & disc) + seg_maps.append(_combine_masks(disc, cup.astype(np.uint8))) + return seg_maps + + +# --------------------------------------------------------------------------- +# Dataset +# --------------------------------------------------------------------------- + +class SegMapDataset(Dataset): + """ + PyTorch Dataset that yields (seg_tensor, label) pairs. + + Parameters + ---------- + records : list of SegMapRecord + target_size : CNN input spatial size (images are resized to this) + channels : 1 = single-channel label map; 3 = one-hot three channels + augment : apply random flips + rotation (for training set) + unet_segmenter : if provided, use U-Net predictions instead of GT masks; + must be a loaded UNetSegmenter with model weights set + unet_threshold : threshold for U-Net logit → binary mask + seg_target_size: resolution at which GT masks are rasterised (or U-Net + output size). Default 512 matches UNetSegmenter default. + crop_to_disc : crop the seg map tightly to the disc bounding box before + resizing to target_size (default True — eliminates the + background zeros that make up most of the full image) + """ + + def __init__( + self, + records: List[SegMapRecord], + target_size: int = 224, + channels: int = 3, + augment: bool = False, + unet_segmenter=None, + unet_threshold: float = 0.5, + seg_target_size: int = 512, + crop_to_disc: bool = True, + precomputed_seg_maps: Optional[List[np.ndarray]] = None, + ) -> None: + self.records = records + self.target_size = target_size + self.channels = channels + self.augment = augment + self.seg_target_size = seg_target_size + self.crop_to_disc = crop_to_disc + + if precomputed_seg_maps is not None: + self._seg_maps = precomputed_seg_maps + elif unet_segmenter is not None: + self._seg_maps = precompute_unet_seg_maps( + records, unet_segmenter, unet_threshold + ) + else: + self._seg_maps = None + + # ------------------------------------------------------------------ + def __len__(self) -> int: + return len(self.records) + + # ------------------------------------------------------------------ + def _augment(self, seg_map: np.ndarray) -> np.ndarray: + """Random flips + 90° rotations (label-safe since NEAREST resize).""" + if np.random.rand() < 0.5: + seg_map = np.fliplr(seg_map) + if np.random.rand() < 0.5: + seg_map = np.flipud(seg_map) + k = np.random.randint(0, 4) + if k: + seg_map = np.rot90(seg_map, k=k) + return np.ascontiguousarray(seg_map) + + # ------------------------------------------------------------------ + def __getitem__(self, idx: int): + rec = self.records[idx] + + if self._seg_maps is not None: + seg_map = self._seg_maps[idx] + else: + disc_mask, cup_mask = load_gt_masks(rec, self.seg_target_size) + seg_map = _combine_masks(disc_mask, cup_mask) + + if self.crop_to_disc: + seg_map = crop_to_disc(seg_map) + + if self.augment: + seg_map = self._augment(seg_map) + + tensor = seg_map_to_tensor(seg_map, self.channels, self.target_size) + return tensor, rec.label + + +# --------------------------------------------------------------------------- +# Model +# --------------------------------------------------------------------------- + +class SegCNN(nn.Module): + """ + Pretrained CNN backbone adapted for segmentation-map input. + + Parameters + ---------- + num_classes : output classes (2 for binary glaucoma grading) + backbone : "resnet18" | "resnet50" | "efficientnet_b0" + pretrained : initialise with ImageNet weights (recommended even for + non-RGB input — transfer generalises across domains) + in_channels : 1 (single label map) or 3 (one-hot channels) + dropout : dropout rate before the final classifier head + """ + + def __init__( + self, + num_classes: int = 2, + backbone: str = "resnet18", + pretrained: bool = True, + in_channels: int = 3, + dropout: float = 0.3, + ) -> None: + super().__init__() + + weights_arg = "DEFAULT" if pretrained else None + + if backbone == "resnet18": + base = models.resnet18(weights=weights_arg) + feat_dim = base.fc.in_features + base.fc = nn.Identity() + elif backbone == "resnet50": + base = models.resnet50(weights=weights_arg) + feat_dim = base.fc.in_features + base.fc = nn.Identity() + elif backbone == "efficientnet_b0": + base = models.efficientnet_b0(weights=weights_arg) + feat_dim = base.classifier[1].in_features + base.classifier = nn.Identity() + else: + raise ValueError(f"Unknown backbone: {backbone!r}") + + # Adapt first conv layer if in_channels ≠ 3 + if in_channels != 3: + first_conv = self._find_first_conv(base) + new_conv = nn.Conv2d( + in_channels, + first_conv.out_channels, + kernel_size=first_conv.kernel_size, + stride=first_conv.stride, + padding=first_conv.padding, + bias=first_conv.bias is not None, + ) + if pretrained: + # Average pretrained RGB weights across channel dim + with torch.no_grad(): + new_conv.weight.copy_( + first_conv.weight.mean(dim=1, keepdim=True).expand_as(new_conv.weight) + ) + self._replace_first_conv(base, new_conv) + + self.backbone = base + self.head = nn.Sequential( + nn.Dropout(p=dropout), + nn.Linear(feat_dim, num_classes), + ) + + # ------------------------------------------------------------------ + @staticmethod + def _find_first_conv(module: nn.Module) -> nn.Conv2d: + for m in module.modules(): + if isinstance(m, nn.Conv2d): + return m + raise RuntimeError("No Conv2d found in backbone") + + @staticmethod + def _replace_first_conv(module: nn.Module, new_conv: nn.Conv2d) -> None: + """Replace the first Conv2d in-place (handles resnet and efficientnet).""" + for name, child in module.named_children(): + if isinstance(child, nn.Conv2d): + setattr(module, name, new_conv) + return + try: + SegCNN._replace_first_conv(child, new_conv) + return + except RuntimeError: + pass + raise RuntimeError("Could not replace first Conv2d") + + # ------------------------------------------------------------------ + def forward(self, x: torch.Tensor) -> torch.Tensor: + feats = self.backbone(x) + if feats.dim() > 2: + feats = feats.flatten(1) + return self.head(feats) diff --git a/v3/classes/split_manager.py b/v3/classes/split_manager.py index 723de75..b2d8e3d 100644 --- a/v3/classes/split_manager.py +++ b/v3/classes/split_manager.py @@ -83,6 +83,65 @@ def build_patient_split_plans( return plans +class EyeLevelSplitManager: + """ + Eye-level (leaky) splitter — splits on individual eye rows, ignoring + patient grouping. Same patient's eyes can appear in different folds. + Used to demonstrate the effect of data leakage. + """ + + def __init__( + self, + *, + patient_col: str = "Patient ID", + label_col: Optional[str] = None, + ) -> None: + self.patient_col = patient_col + self.label_col = label_col + + def build_plans( + self, + *, + clinical: Any, + args: Any, + profile: Optional[Any] = None, + ) -> list[PatientSplit]: + profile_label_col = getattr(profile, "label_col", None) if profile is not None else None + label_col = self.label_col or profile_label_col or getattr(clinical, "label_col", None) + if label_col is None: + raise ValueError("Could not resolve label column") + + if not hasattr(clinical, "df"): + raise ValueError("Clinical object must expose a dataframe at .df") + df_full = clinical.df.copy().reset_index(drop=True) + + eval_mode = str(getattr(args, "eval_mode", "multiclass")).lower() + if eval_mode == "binary": + df_full = df_full[df_full[label_col].isin([0, 1])].reset_index(drop=True) + + n_splits = int(getattr(args, "n_splits", 5)) + fold_seed = int(getattr(args, "fold_seed", 42)) + + labels = df_full[label_col].to_numpy() + eye_ids = df_full.index.to_numpy() + + # Reuse build_patient_split_plans with eye-row IDs as the "patients" + plans = build_patient_split_plans( + patient_ids=eye_ids, + patient_labels=labels, + n_splits=n_splits, + seed=fold_seed, + ) + + out: list[PatientSplit] = [] + for plan in plans: + train_df = df_full[df_full.index.isin(plan.train_patient_ids)].reset_index(drop=True) + val_df = df_full[df_full.index.isin(plan.val_patient_ids)].reset_index(drop=True) + test_df = df_full[df_full.index.isin(plan.test_patient_ids)].reset_index(drop=True) + out.append(PatientSplit(train=train_df, val=val_df, test=test_df)) + return out + + class PatientFirstSplitManager: """Patient-level splitter for V3. Outer/inner k-fold, no holdout.""" diff --git a/v3/classes/v3_hypertower.py b/v3/classes/v3_hypertower.py index a11f9cb..0cb6785 100644 --- a/v3/classes/v3_hypertower.py +++ b/v3/classes/v3_hypertower.py @@ -23,7 +23,11 @@ from typing import Optional import numpy as np import torch -from v3.classes.croppers import build_image_preprocessor_from_args +from v3.classes.croppers import ( + ManifestImageCropper, + UNetImageCropper, + build_image_preprocessor_from_args, +) from v3.classes.image_loader import CachedImageLoader from v3.classes.dataset import _ClinicalView # noqa: F401 from v3.classes.loader_factory import ( @@ -35,10 +39,14 @@ from v3.classes.loader_factory import ( from v3.classes.metrics import _score_arrays, _svf, _tune_and_snap from v3.classes.models import ( BilateralHT, + EmbeddingMLPEnsembleHT, FusedEnsembleHT, + LogitMLPEnsembleHT, + SiameseHT, SingleEyeHT, V2ModeComparisonOps, collect_probs_bilateral, + collect_probs_siamese, collect_probs_bilateral_components, collect_probs_classic, collect_probs_ensemble, @@ -47,6 +55,7 @@ from v3.classes.models import ( collect_probs_fused, collect_probs_single_components, train_bilateral_epoch, + train_siamese_epoch, train_fusion_epoch, train_single_epoch, ) @@ -54,7 +63,7 @@ from v3.classes.papila_builders import build_papila_data from v3.classes.predictions import PredictionStore, head_names_for_mode from v3.classes.profiles import build_papila_profile from v3.classes.results import FoldArtifacts, FoldResult, _f, _nan, _sv -from v3.classes.split_manager import PatientFirstSplitManager +from v3.classes.split_manager import EyeLevelSplitManager, PatientFirstSplitManager from v3.classes.transforms import build_eval_transform from v3.classes.utils import ( _drop_mixed_label_patients, @@ -145,11 +154,14 @@ class V3HyperTower: ap.add_argument("--exclude-cols", nargs="*", default=[]) ap.add_argument("--eval-mode", choices=["binary", "multiclass"], default="binary") ap.add_argument( - "--tower-mode", choices=["single", "ensemble", "bilateral", "classic"], + "--tower-mode", choices=["single", "ensemble", "bilateral", "siamese", "classic"], default="ensemble", ) ap.add_argument("--n-splits", type=int, default=5) ap.add_argument("--fold-seed", type=int, default=42) + ap.add_argument("--leaky-cv", action="store_true", + help="Split at eye level (leaky: same patient can span folds). " + "Used to demonstrate data-leakage effect.") ap.add_argument( "--folds", type=int, default=None, help="Optional cap on number of folds to run.", @@ -157,9 +169,9 @@ class V3HyperTower: ap.add_argument("--epochs", type=int, default=40) ap.add_argument("--warmup-tower-epochs", type=int, default=None) ap.add_argument("--warmup-fused-epochs", type=int, default=None) - ap.add_argument("--single-warmup-tower-epochs", type=int, default=None) - ap.add_argument("--single-warmup-fused-epochs", type=int, default=None) - ap.add_argument("--warmup-cd-epochs", type=int, default=0) + ap.add_argument("--single-warmup-tower-epochs", type=int, default=3) + ap.add_argument("--single-warmup-fused-epochs", type=int, default=3) + ap.add_argument("--warmup-cd-epochs", type=int, default=40) ap.add_argument("--bilat-warmup-tower-epochs", type=int, default=None) ap.add_argument("--bilat-warmup-fused-epochs", type=int, default=None) ap.add_argument("--batch-size", type=int, default=8) @@ -170,7 +182,7 @@ class V3HyperTower: ap.add_argument("--freeze-ratio", type=float, default=0.0) ap.add_argument("--augment", action="store_true") ap.add_argument("--balanced-sampling", action="store_true") - ap.add_argument("--num-workers", type=int, default=4) + ap.add_argument("--num-workers", type=int, default=8) ap.add_argument("--in-memory-cache", action="store_true", default=True) ap.add_argument("--no-in-memory-cache", action="store_false", dest="in_memory_cache") ap.add_argument("--cache-workers", type=int, default=4) @@ -191,10 +203,20 @@ class V3HyperTower: ap.add_argument("--img-crop-cache", type=str, default="cache_data/hypertower_crops") ap.add_argument("--persist-img-crop-cache", action="store_true") # Architecture - ap.add_argument("--cd-hidden-dim", type=int, default=128) - ap.add_argument("--fusion-dim", type=int, default=256) - ap.add_argument("--bridge-mode", default="fused", + ap.add_argument("--cd-hidden-dim", type=int, default=128) + ap.add_argument("--fusion-dim", type=int, default=256) + ap.add_argument("--bridge-mode", default="fused", choices=["fused", "image_only", "clinical_only"]) + ap.add_argument("--bridge-dropout", type=float, default=0.5, + help="Dropout in bridge classifier_fused (default: 0.5)") + ap.add_argument("--cd-dropout", type=float, default=0.1, + help="Dropout in clinical tower MLP (default: 0.1)") + ap.add_argument("--se-img-tower", action="store_true", + help="Enable SE gate on image tower output features") + ap.add_argument("--se-cd-tower", action="store_true", + help="Enable SE gate on clinical tower output features") + ap.add_argument("--se-bridge", action="store_true", + help="Enable SE gate on fused vector inside the bridge") # Mixed patients ap.add_argument("--exclude-mixed-patients", dest="exclude_mixed_patients", action="store_true") @@ -217,6 +239,19 @@ class V3HyperTower: # Fused head ap.add_argument("--fused-head", action="store_true") ap.add_argument("--fusion-epochs", type=int, default=10) + ap.add_argument("--head-type", + choices=["attention", "logit_mlp", "embedding_mlp"], + default="attention", + help="Which bilateral head to train on top of frozen ensemble base") + ap.add_argument("--save-checkpoints", action="store_true", + help="Save best_single.pt per fold for explainability / GradCAM") + # Geometry features + ap.add_argument("--geometry-dim", type=int, default=0, + help="Append N geometry features to clinical metadata (0=disabled, 5=all). " + "Requires --img-crop-manifest.") + ap.add_argument("--geometry-source", default="gt", choices=["gt", "unet"], + help="Source for geometry features: gt (GT contour annotations) or " + "unet (U-Net segmentation). unet also requires --img-crop-weights.") return ap def __init__(self, args) -> None: @@ -246,6 +281,49 @@ class V3HyperTower: patient_col="Patient ID", label_col=args.label_col, sample_mode="patient" ) + # Build geometry provider if requested, extend feature_dim to include geometry. + # Both ManifestImageCropper and UNetImageCropper already have geometry_features() + # and precompute_geometry() — we just pick the right one and pre-compute upfront. + self.geometry_provider = None + geom_dim = int(getattr(args, "geometry_dim", 0)) + if geom_dim > 0: + source = getattr(args, "geometry_source", "gt") + manifest = getattr(args, "img_crop_manifest", None) + if not manifest: + raise ValueError("--geometry-dim requires --img-crop-manifest") + all_paths = [ + self.data.get_image_path(row) + for _, row in self.data.df.iterrows() + ] + if source == "gt": + # Reuse image_preprocessor if it's already a ManifestImageCropper, + # otherwise build a lightweight one just for geometry (no crop cache). + if isinstance(self.image_preprocessor, ManifestImageCropper): + provider = self.image_preprocessor + else: + provider = ManifestImageCropper(manifest_path=Path(manifest)) + print(f"[geometry] GT source — pre-computing geometry from {manifest}", flush=True) + elif source == "unet": + weights = getattr(args, "img_crop_weights", None) + if not weights: + raise ValueError("--geometry-source unet requires --img-crop-weights") + if isinstance(self.image_preprocessor, UNetImageCropper): + provider = self.image_preprocessor + else: + provider = UNetImageCropper( + manifest_path=Path(manifest), + weights_path=Path(weights), + normalize=getattr(args, "img_crop_normalize", "per_image"), + threshold=getattr(args, "img_crop_threshold", 0.5), + ) + print(f"[geometry] UNet source — pre-computing geometry from {weights}", flush=True) + else: + raise ValueError(f"Unknown --geometry-source: {source!r}") + provider.precompute_geometry(all_paths) + self.geometry_provider = provider + self.data.feature_dim += geom_dim + print(f"[geometry] feature_dim extended to {self.data.feature_dim} (+{geom_dim} geometry)", flush=True) + def run(self) -> Path: """Execute the full fold loop.""" args = self.args @@ -276,7 +354,11 @@ class V3HyperTower: num_classes = 2 if mode == "binary" else int(df_mode[args.label_col].nunique()) print(f"\n[{mode}] num_classes={num_classes} rows={len(df_mode)} patients={df_mode['Patient ID'].nunique()}", flush=True) - split_manager = PatientFirstSplitManager(patient_col="Patient ID", label_col=args.label_col) + if getattr(args, "leaky_cv", False): + split_manager = EyeLevelSplitManager(patient_col="Patient ID", label_col=args.label_col) + print("[CV] WARNING: leaky-cv mode — eye-level splits, same patient can span folds.", flush=True) + else: + split_manager = PatientFirstSplitManager(patient_col="Patient ID", label_col=args.label_col) split_args = SimpleNamespace( eval_mode=mode, n_splits=args.n_splits, @@ -390,7 +472,7 @@ class V3HyperTower: _test_key = "classic_test" elif tower_mode == "ensemble": _test_key = "ensemble_test" - elif tower_mode == "bilateral": + elif tower_mode in ("bilateral", "siamese"): _test_key = "bilat_test" else: _test_key = "classic_test" @@ -400,6 +482,25 @@ class V3HyperTower: return out_dir + def _augment_geometry(self, samples: list) -> list: + """Append geometry features to matrix_1/matrix_2 in each sample dict.""" + if self.geometry_provider is None: + return samples + geom_dim = int(getattr(self.args, "geometry_dim", 0)) + for s in samples: + for img_slot, mat_slot in (("image_1", "matrix_1"), ("image_2", "matrix_2")): + img_path = s.get(img_slot) + mat = s.get(mat_slot) + if img_path is None or mat is None: + continue + vec = self.geometry_provider.geometry_for_image(img_path) + if vec is not None and len(vec) >= geom_dim: + geom = vec[:geom_dim].astype(np.float32) + else: + geom = np.zeros(geom_dim, dtype=np.float32) + s[mat_slot] = np.concatenate([np.asarray(mat, dtype=np.float32), geom]) + return samples + def _run_fold( self, *, @@ -420,9 +521,10 @@ class V3HyperTower: image_preprocessor = self.image_preprocessor nan = float("nan") - run_single = tower_mode in ("single", "ensemble") - run_bilat = tower_mode == "bilateral" - run_fused = tower_mode == "ensemble" and getattr(args, "fused_head", False) + run_single = tower_mode in ("single", "ensemble") + run_bilat = tower_mode == "bilateral" + run_siamese = tower_mode == "siamese" + run_fused = tower_mode == "ensemble" and getattr(args, "fused_head", False) # ---- warmup schedule ------------------------------------------- global_warmup_tower = getattr(args, "warmup_tower_epochs", None) @@ -442,7 +544,7 @@ class V3HyperTower: single_warmup_cd = int(getattr(args, "warmup_cd_epochs", 0)) if run_single else 0 if not run_single: single_warmup_tower = single_warmup_fused = 0 - if not run_bilat: + if not run_bilat and not run_siamese: bilat_warmup_tower = bilat_warmup_fused = 0 # Warmup is meaningless in single-pathway modes — skip it entirely _bridge_mode = getattr(args, "bridge_mode", "fused") @@ -451,7 +553,7 @@ class V3HyperTower: bilat_warmup_tower = bilat_warmup_fused = 0 main_epochs = int(args.epochs) total_single_epochs = (single_warmup_cd + single_warmup_tower + single_warmup_fused + main_epochs) if run_single else 0 - total_bilat_epochs = (bilat_warmup_tower + bilat_warmup_fused + main_epochs) if run_bilat else 0 + total_bilat_epochs = (bilat_warmup_tower + bilat_warmup_fused + main_epochs) if (run_bilat or run_siamese) else 0 total_epochs = max(total_single_epochs, total_bilat_epochs) # ---- samples --------------------------------------------------- @@ -460,6 +562,12 @@ class V3HyperTower: bilat_val = filter_bilateral_samples(profile_patient.build_samples(df=split.val, clinical=data)) bilat_test = filter_bilateral_samples(profile_patient.build_samples(df=split.test, clinical=data)) if split.test is not None else [] + if self.geometry_provider is not None: + eye_train = self._augment_geometry(eye_train) + bilat_train = self._augment_geometry(bilat_train) + bilat_val = self._augment_geometry(bilat_val) + bilat_test = self._augment_geometry(bilat_test) + if pred_store is not None: if tower_mode in ("single", "classic"): train_sids = [f"{s['id_1']}{s.get('eye_id_1','')}" for s in eye_train] @@ -502,27 +610,41 @@ class V3HyperTower: y_true_bilat=None, probs_bilat=None, ) - # ---- models ---------------------------------------------------- + # ---- models (CPU for now — moved to device after workers spawn) --- single = None bilateral = None + siamese = None if run_single: single = SingleEyeHT( backbone=args.backbone, freeze_ratio=args.freeze_ratio, augment=args.augment, clinical_data=data, num_classes=num_classes, cd_hidden_dim=args.cd_hidden_dim, fusion_dim=args.fusion_dim, bridge_mode=getattr(args, "bridge_mode", "fused"), - ).to(device) + bridge_dropout=getattr(args, "bridge_dropout", 0.5), + cd_dropout=getattr(args, "cd_dropout", 0.1), + se_img_tower=getattr(args, "se_img_tower", False), + se_cd_tower=getattr(args, "se_cd_tower", False), + se_bridge=getattr(args, "se_bridge", False), + ) if run_bilat: bilateral = BilateralHT( backbone=args.backbone, freeze_ratio=args.freeze_ratio, augment=args.augment, clinical_data=data, num_classes=num_classes, cd_hidden_dim=args.cd_hidden_dim, fusion_dim=args.fusion_dim, - ).to(device) + ) + if run_siamese: + siamese = SiameseHT( + backbone=args.backbone, freeze_ratio=args.freeze_ratio, + augment=args.augment, num_classes=num_classes, + fusion_dim=args.fusion_dim, + ) slots_eye = profile_eye.slot_descriptors() slots_patient = profile_patient.slot_descriptors() + _persistent_workers = args.num_workers > 0 loader_kw = dict(batch_size=args.batch_size, num_workers=args.num_workers, - image_cache=image_cache) + image_cache=image_cache, + persistent_workers=_persistent_workers) # ---- loaders --------------------------------------------------- use_balanced = bool(getattr(args, "balanced_sampling", False)) @@ -553,6 +675,13 @@ class V3HyperTower: image_preprocessor=image_preprocessor, shuffle=True, sampler=bilat_sampler, **loader_kw, ) + elif run_siamese: + siamese_sampler = build_balanced_sampler(bilat_train) if use_balanced else None + train_bilat_loader = make_loader( + bilat_train, slots_patient, image_transform=siamese.transform, + image_preprocessor=image_preprocessor, shuffle=True, + sampler=siamese_sampler, **loader_kw, + ) elif run_fused: fused_sampler = build_balanced_sampler(bilat_train) if use_balanced else None train_bilat_loader = make_loader( @@ -581,8 +710,26 @@ class V3HyperTower: if _ldr is not None: _ldr.dataset.prebuild_image_cache() - opt_single = torch.optim.Adam(single.parameters(), lr=args.lr) if run_single else None - opt_bilateral = torch.optim.Adam(bilateral.parameters(), lr=args.lr) if run_bilat else None + # ---- spawn DataLoader workers BEFORE CUDA init ----------------- + # Workers fork here (clean process state, no CUDA context yet). + # persistent_workers=True keeps them alive so the training loop + # reuses them rather than re-forking after .to(device). + if _persistent_workers: + for _ldr in [train_single_loader, train_bilat_loader, val_loader, test_loader]: + if _ldr is not None: + _ = iter(_ldr) # triggers fork now, before CUDA + + # ---- move models to device (CUDA init happens here) ------------ + if single is not None: + single = single.to(device) + if bilateral is not None: + bilateral = bilateral.to(device) + if siamese is not None: + siamese = siamese.to(device) + + opt_single = torch.optim.Adam(single.parameters(), lr=args.lr) if run_single else None + opt_bilateral = torch.optim.Adam(bilateral.parameters(), lr=args.lr) if run_bilat else None + opt_siamese = torch.optim.Adam(siamese.parameters(), lr=args.lr) if run_siamese else None # ---- epoch log ------------------------------------------------- epoch_fields = [ @@ -679,7 +826,7 @@ class V3HyperTower: else: phase_single, main_epoch_single, single_active = "done", main_epochs, False - if not run_bilat: + if not run_bilat and not run_siamese: phase_bilat, main_epoch_bilat, bilat_active = "inactive", 0, False elif epoch < bilat_warmup_tower: phase_bilat, main_epoch_bilat, bilat_active = "tower_warmup", 0, True @@ -709,6 +856,12 @@ class V3HyperTower: phase=phase_bilat, bcd_prob=float(args.bcd_prob), tower_loss_mode=args.tower_loss_mode, ) + elif run_siamese and bilat_active: + bl_loss, bl_acc = train_siamese_epoch( + siamese, train_bilat_loader, opt_siamese, device, + bcd_prob=float(args.bcd_prob), + tower_loss_mode=args.tower_loss_mode, + ) else: bl_loss, bl_acc = nan, nan @@ -763,6 +916,10 @@ class V3HyperTower: bi_acc_cd = float((p_bi_cd.argmax(1) ==y_bi).mean()) if y_bi.size else nan _, bi_auc_img, _ = _score_arrays(y_bi, p_bi_img, num_classes) _, bi_auc_cd, _ = _score_arrays(y_bi, p_bi_cd, num_classes) + elif run_siamese and not _skip_val_eval: + y_bi, p_bi = collect_probs_siamese(siamese, val_loader, device) + bi_acc, bi_auc, bi_n = _score_arrays(y_bi, p_bi, num_classes) + bi_acc_img = bi_acc_cd = bi_auc_img = bi_auc_cd = nan else: y_bi = np.array([], dtype=np.int64) p_bi = np.zeros((0, 0), dtype=np.float32) @@ -905,7 +1062,7 @@ class V3HyperTower: _bar_w = 30 _filled = int(_bar_w * (epoch + 1) / single_warmup_cd) _bar = "#" * _filled + "-" * (_bar_w - _filled) - msg = f" [fold {fold+1}] md_warmup [{_bar}] {epoch+1}/{single_warmup_cd} loss={sl_loss:.4f}" + msg = f" [fold {fold+1}] md_warmup [{_bar}] {epoch+1}/{single_warmup_cd} loss={sl_loss:.2f}" print(f"\r{msg}", end="", flush=True) fold_logger.info(msg) _prev_phase_single = phase_single @@ -917,13 +1074,13 @@ class V3HyperTower: if args.log_every > 0 and (epoch + 1) % args.log_every == 0: _epoch_secs = time.time() - _epoch_t0 if tower_mode == "ensemble": - msg = f" ep {epoch+1:>3}/{total_epochs} ({_epoch_secs:.1f}s) auc={en_auc:.4f} acc={en_acc:.4f}" - elif tower_mode == "bilateral": - msg = f" ep {epoch+1:>3}/{total_epochs} ({_epoch_secs:.1f}s) auc={bi_auc:.4f} acc={bi_acc:.4f}" + _auc_v, _acc_v = en_auc, en_acc + elif tower_mode in ("bilateral", "siamese"): + _auc_v, _acc_v = bi_auc, bi_acc else: - msg = f" ep {epoch+1:>3}/{total_epochs} ({_epoch_secs:.1f}s) auc={cl_auc:.4f} acc={cl_acc:.4f}" - print(msg, flush=True) - fold_logger.info(msg) + _auc_v, _acc_v = cl_auc, cl_acc + print(f" ep {epoch+1:>3}/{total_epochs} ({_epoch_secs:.1f}s) auc={_auc_v:.2f} acc={_acc_v:.2f}", flush=True) + fold_logger.info(f" ep {epoch+1:>3}/{total_epochs} ({_epoch_secs:.1f}s) auc={_auc_v:.4f} acc={_acc_v:.4f}") _prev_phase_single = phase_single @@ -958,8 +1115,14 @@ class V3HyperTower: if run_fused and single is not None: for p in single.parameters(): p.requires_grad_(False) - fused = FusedEnsembleHT(single, num_classes).to(device) - opt_fused = torch.optim.Adam(fused.eye_scorer.parameters(), lr=args.lr) + head_type = getattr(args, "head_type", "attention") + if head_type == "logit_mlp": + fused = LogitMLPEnsembleHT(single, num_classes).to(device) + elif head_type == "embedding_mlp": + fused = EmbeddingMLPEnsembleHT(single, num_classes).to(device) + else: + fused = FusedEnsembleHT(single, num_classes).to(device) + opt_fused = torch.optim.Adam(fused.head.parameters(), lr=args.lr) fusion_epochs = int(getattr(args, "fusion_epochs", 10)) print( f" [fold {fold+1}] Phase 2: fusion head bilat_train_n={len(bilat_train)} epochs={fusion_epochs}", @@ -977,8 +1140,8 @@ class V3HyperTower: snap_fused, _, _, _ = _tune_and_snap(y_fu, p_fu, fu_acc_val, num_classes, args, args.ece_bins) if (fep + 1) % max(1, args.log_every) == 0: print( - f" [fold {fold+1}] fusion ep{fep+1:>3} loss={fu_loss:.4f} " - f"val_auc={fu_auc:.4f}", + f" [fold {fold+1}] fusion ep{fep+1:>3} loss={fu_loss:.2f} " + f"val_auc={fu_auc:.2f}", flush=True, ) # No checkpoint saving for fused head either. @@ -1015,9 +1178,16 @@ class V3HyperTower: p_cl_best_img = p_cl_best_md = None if run_bilat: y_bi_best, p_bi_best = collect_probs_bilateral(bilateral, val_loader, device) + elif run_siamese: + y_bi_best, p_bi_best = collect_probs_siamese(siamese, val_loader, device) else: y_bi_best = p_bi_best = None + # Optional checkpoint saving (final-epoch weights for explainability) + if getattr(args, "save_checkpoints", False) and run_single and single is not None: + import torch as _torch + _torch.save(single.state_dict(), fold_dir / "best_single.pt") + # Compute val snaps from final-epoch model state if run_single and tower_mode == "single" and y_cl_best is not None: snap_cl, _, _, _ = _tune_and_snap(y_cl_best, p_cl_best, float((p_cl_best.argmax(1) == y_cl_best).mean()), num_classes, args, args.ece_bins) @@ -1025,7 +1195,7 @@ class V3HyperTower: elif run_single and tower_mode == "ensemble" and y_en_best is not None: snap_en, _, _, _ = _tune_and_snap(y_en_best, p_en_best, float((p_en_best.argmax(1) == y_en_best).mean()), num_classes, args, args.ece_bins) snap_ensemble = snap_en - if run_bilat and y_bi_best is not None: + if (run_bilat or run_siamese) and y_bi_best is not None: snap_bi, _, _, _ = _tune_and_snap(y_bi_best, p_bi_best, float((p_bi_best.argmax(1) == y_bi_best).mean()), num_classes, args, args.ece_bins) snap_bilat = snap_bi @@ -1050,6 +1220,8 @@ class V3HyperTower: y_test_out, p_test_out, _, _ = collect_probs_bilateral_components( bilateral, test_loader, device ) + elif run_siamese: + y_test_out, p_test_out = collect_probs_siamese(siamese, test_loader, device) if y_test_out is not None and y_test_out.size: test_acc_raw = float((p_test_out.argmax(1) == y_test_out).mean()) snap_test, _, _, _ = _tune_and_snap( @@ -1057,11 +1229,11 @@ class V3HyperTower: ) print( f" [fold {fold+1}] TEST " - f"auc={snap_test.get('auc', nan):.4f} " - f"acc={snap_test.get('acc', nan):.4f} " - f"kappa={snap_test.get('kappa', nan):.4f} " - f"f1={snap_test.get('macro_f1', nan):.4f} " - f"ece={snap_test.get('ece', nan):.4f} " + f"auc={snap_test.get('auc', nan):.2f} " + f"acc={snap_test.get('acc', nan):.2f} " + f"kappa={snap_test.get('kappa', nan):.2f} " + f"f1={snap_test.get('macro_f1', nan):.2f} " + f"ece={snap_test.get('ece', nan):.2f} " f"n={snap_test.get('n', 0)}", flush=True, ) @@ -1118,11 +1290,11 @@ class V3HyperTower: classic_test_kappa=snap_test.get("kappa", nan) if tower_mode == "single" else nan, classic_test_f1=snap_test.get("macro_f1", nan) if tower_mode == "single" else nan, classic_test_ece=snap_test.get("ece", nan) if tower_mode == "single" else nan, - bilat_test_auc=snap_test.get("auc", nan) if tower_mode == "bilateral" else nan, - bilat_test_acc=snap_test.get("acc", nan) if tower_mode == "bilateral" else nan, - bilat_test_kappa=snap_test.get("kappa", nan) if tower_mode == "bilateral" else nan, - bilat_test_f1=snap_test.get("macro_f1", nan) if tower_mode == "bilateral" else nan, - bilat_test_ece=snap_test.get("ece", nan) if tower_mode == "bilateral" else nan, + bilat_test_auc=snap_test.get("auc", nan) if tower_mode in ("bilateral", "siamese") else nan, + bilat_test_acc=snap_test.get("acc", nan) if tower_mode in ("bilateral", "siamese") else nan, + bilat_test_kappa=snap_test.get("kappa", nan) if tower_mode in ("bilateral", "siamese") else nan, + bilat_test_f1=snap_test.get("macro_f1", nan) if tower_mode in ("bilateral", "siamese") else nan, + bilat_test_ece=snap_test.get("ece", nan) if tower_mode in ("bilateral", "siamese") else nan, test_n=test_n, single_train_n=len(eye_train), bilat_train_n=len(bilat_train), @@ -1222,18 +1394,18 @@ class V3HyperTower: @staticmethod def _print_summary(mode: str, s: dict, tower_mode: str | None = None) -> None: def f(v): - return " nan " if v is None else f"{v:.4f}" + return " nan " if v is None else f"{v:.2f}" def fsd(mean, std): - if mean is None: return " nan " - if std is None: return f"{mean:.4f} " - return f"{mean:.4f}±{std:.4f}" + if mean is None: return " nan " + if std is None: return f"{mean:.2f} " + return f"{mean:.2f}±{std:.2f}" # Resolve test key if tower_mode in ("single", "classic"): test_key = "classic_test" elif tower_mode == "ensemble": test_key = "ensemble_test" - elif tower_mode == "bilateral": + elif tower_mode in ("bilateral", "siamese"): test_key = "bilat_test" else: test_key = "classic_test" diff --git a/v3/distributed/__init__.py b/v3/distributed/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/v3/distributed/cli.py b/v3/distributed/cli.py new file mode 100644 index 0000000..69c1b20 --- /dev/null +++ b/v3/distributed/cli.py @@ -0,0 +1,379 @@ +""" +HyperTower distributed job CLI — submit jobs, view status. + +Usage: + # View all connected clients + python -m v3.distributed.cli clients + + # View a specific client + python -m v3.distributed.cli clients + + # View jobs (optionally filter by state) + python -m v3.distributed.cli jobs [--state pending|running|done|failed] + + # Submit a job + python -m v3.distributed.cli submit \\ + --run-name phase2/leaky \\ + -- --bridge-mode image_only --backbone resnet50 --reps 10 ... + + # Submit all jobs from a batch file (JSON) + python -m v3.distributed.cli submit-batch jobs.json + + # Cancel a pending job + python -m v3.distributed.cli cancel + +Global flags (can also be set via env vars): + --server HT_SERVER e.g. http://apollo:8765 + --token HT_TOKEN +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from datetime import datetime, timezone, timedelta +from typing import Optional + +import requests + +# ────────────────────────────────────────────────────────────── +# HTTP helpers +# ────────────────────────────────────────────────────────────── + +class _API: + def __init__(self, base_url: str, token: str): + self.base_url = base_url.rstrip("/") + self._h = {"x-token": token} + + def get(self, path: str, **params) -> object: + r = requests.get(f"{self.base_url}{path}", headers=self._h, + params=params, timeout=10) + r.raise_for_status() + return r.json() + + def post(self, path: str, body: dict) -> object: + r = requests.post(f"{self.base_url}{path}", headers=self._h, + json=body, timeout=10) + r.raise_for_status() + return r.json() + + def delete(self, path: str) -> object: + r = requests.delete(f"{self.base_url}{path}", headers=self._h, timeout=10) + r.raise_for_status() + return r.json() + + +# ────────────────────────────────────────────────────────────── +# Formatting helpers +# ────────────────────────────────────────────────────────────── + +def _ago(ts: Optional[str]) -> str: + if not ts: + return "-" + try: + dt = datetime.fromisoformat(ts) + delta = datetime.now(timezone.utc) - dt + secs = int(delta.total_seconds()) + if secs < 60: + return f"{secs}s ago" + elif secs < 3600: + return f"{secs//60}m ago" + else: + return f"{secs//3600}h{(secs%3600)//60}m ago" + except Exception: + return ts + + +def _col(text: str, width: int) -> str: + text = str(text) if text is not None else "-" + return text[:width].ljust(width) + + +def _table(rows: list[list[str]], headers: list[str]): + widths = [max(len(str(r[i])) for r in ([headers] + rows)) for i in range(len(headers))] + sep = " " + def _row(r): + return sep.join(str(r[i]).ljust(widths[i]) for i in range(len(r))) + print(_row(headers)) + print("-" * (sum(widths) + len(sep) * (len(widths) - 1))) + for r in rows: + print(_row(r)) + + +# ────────────────────────────────────────────────────────────── +# Subcommands +# ────────────────────────────────────────────────────────────── + +def _clients_table(api: _API): + """Render one clients snapshot. Returns the printed lines as a string.""" + clients = api.get("/clients") + if not clients: + return "No clients connected." + rows = [] + for c in clients: + s = c["status"] + parts = [] + if s.get("rep") is not None: + parts.append(f"rep{s['rep']:02d}") + if s.get("fold") is not None: + parts.append(f"f{s['fold']}") + if s.get("epoch") is not None: + parts.append(f"ep{s['epoch']}/{s.get('total_epochs','?')}") + parts.append(f"auc={s.get('last_auc','?')}") + prog = " ".join(parts) if parts else "-" + rows.append([ + c["client_id"], + c["hostname"], + c["gpu_info"][:30], + s["state"], + s.get("run_name") or "-", + prog, + _ago(c["last_seen"]), + ]) + headers = ["ID", "HOST", "GPU", "STATE", "RUN", "PROGRESS", "SEEN"] + widths = [max(len(str(r[i])) for r in ([headers] + rows)) for i in range(len(headers))] + sep = " " + lines = [] + lines.append(sep.join(str(h).ljust(widths[i]) for i, h in enumerate(headers))) + lines.append("-" * (sum(widths) + len(sep) * (len(widths) - 1))) + for r in rows: + lines.append(sep.join(str(r[i]).ljust(widths[i]) for i in range(len(r)))) + return "\n".join(lines) + + +def cmd_clients(api: _API, args): + if hasattr(args, "client_id") and args.client_id: + data = api.get(f"/clients/{args.client_id}") + s = data["status"] + print(f"client_id : {data['client_id']}") + print(f"hostname : {data['hostname']}") + print(f"gpu : {data['gpu_info']}") + print(f"last_seen : {_ago(data['last_seen'])}") + print(f"state : {s['state']}") + if s.get("job_id"): + print(f"job : {s['job_id']} ({s.get('run_name', '')})") + if s.get("rep") is not None: + print(f"progress : rep {s['rep']} fold {s.get('fold', '?')} " + f"ep {s.get('epoch', '?')}/{s.get('total_epochs', '?')} " + f"({s.get('last_epoch_secs', '?')}s/ep) " + f"auc={s.get('last_auc', '?')}") + return + + watch = getattr(args, "watch", False) + interval = getattr(args, "interval", 5) + + if not watch: + print(_clients_table(api)) + return + + try: + while True: + now = datetime.now().strftime("%H:%M:%S") + print(f"\033[H\033[2J", end="") # clear screen + print(f"HyperTower clients [{now}] (Ctrl-C to exit)\n") + print(_clients_table(api)) + time.sleep(interval) + except KeyboardInterrupt: + print("\nStopped.") + + +def _rep_label(job: dict) -> str: + """Extract --rep-index from job args if present.""" + try: + a = json.loads(job["args"]) if isinstance(job.get("args"), str) else job.get("args", []) + if "--rep-index" in a: + idx = a[a.index("--rep-index") + 1] + return f"rep{int(idx):02d}" + except Exception: + pass + return "-" + + +def cmd_jobs(api: _API, args): + params = {} + if hasattr(args, "state") and args.state: + params["state"] = args.state + jobs = api.get("/jobs", **params) + if not jobs: + print("No jobs.") + return + rows = [] + for j in jobs: + attempts = j.get("attempts", 0) + rows.append([ + j["job_id"][:12], + j["run_name"], + _rep_label(j), + j["state"], + f"{attempts}" if attempts else "-", + j.get("assigned_to") or "-", + _ago(j["created_at"]), + _ago(j.get("started_at")), + _ago(j.get("completed_at")), + ]) + _table(rows, ["JOB_ID", "RUN_NAME", "REP", "STATE", "TRIES", "CLIENT", "CREATED", "STARTED", "DONE"]) + pending = sum(1 for j in jobs if j["state"] == "pending") + running = sum(1 for j in jobs if j["state"] == "running") + done = sum(1 for j in jobs if j["state"] == "done") + failed = sum(1 for j in jobs if j["state"] == "failed") + print(f"\n {len(jobs)} total | {pending} pending {running} running {done} done {failed} failed") + + +def cmd_submit(api: _API, args): + body = { + "run_name": args.run_name, + "module": args.module, + "args": args.run_args, + "output_dir": args.output_dir, + "priority": args.priority, + } + resp = api.post("/jobs", body) + print(f"Queued job {resp['job_id']} ({args.run_name})") + + +def cmd_submit_batch(api: _API, args): + with open(args.batch_file) as f: + jobs = json.load(f) + for job in jobs: + resp = api.post("/jobs", job) + print(f"Queued {resp['job_id']} ({job['run_name']})") + + +def cmd_submit_cv(api: _API, args): + """Submit one job per rep, each writing to repNN under the same run-name.""" + seed_start = args.rep_seed_start + seed_step = args.rep_seed_step + submitted = [] + for i in range(args.reps): + seed = seed_start + i * seed_step + rep_args = [a for a in args.run_args + if a not in ("--reps", "--rep-seed-start", "--rep-seed-step")] + rep_args += [ + "--reps", "1", + "--rep-seed-start", str(seed), + "--rep-index", str(i), + ] + body = { + "run_name": args.run_name, + "module": args.module, + "args": rep_args, + "output_dir": args.output_dir, + "priority": args.priority, + } + resp = api.post("/jobs", body) + submitted.append(resp["job_id"]) + print(f"Queued rep{i:02d} seed={seed} job_id={resp['job_id']}") + print(f"\n{len(submitted)} jobs queued for run '{args.run_name}'") + + +def cmd_cancel(api: _API, args): + resp = api.delete(f"/jobs/{args.job_id}") + print(f"Cancelled {args.job_id}" if resp.get("ok") else resp) + + +def cmd_clear(api: _API, args): + body: dict = {} + if args.all: + body["all"] = True + elif args.run_name: + body["run_name"] = args.run_name + else: + body["states"] = args.states or ["done", "failed", "cancelled"] + resp = api.post("/jobs/clear", body) + print(f"Cleared {resp['cleared']} jobs.") + + +# ────────────────────────────────────────────────────────────── +# Parser +# ────────────────────────────────────────────────────────────── + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""), + help="Server URL (or set HT_SERVER)") + ap.add_argument("--token", default=os.environ.get("HT_TOKEN", ""), + help="Shared secret (or set HT_TOKEN)") + + sub = ap.add_subparsers(dest="cmd", required=True) + + # clients + p_cl = sub.add_parser("clients", help="List clients or inspect one") + p_cl.add_argument("client_id", nargs="?") + p_cl.add_argument("--watch", "-w", action="store_true", + help="Live monitoring mode — refresh every --interval seconds") + p_cl.add_argument("--interval", "-n", type=int, default=5, + help="Refresh interval in seconds for --watch (default: 5)") + + # jobs + p_j = sub.add_parser("jobs", help="List jobs") + p_j.add_argument("--state", choices=["pending", "running", "done", "failed", "cancelled"]) + + # submit + p_s = sub.add_parser("submit", help="Submit a single job") + p_s.add_argument("--run-name", required=True) + p_s.add_argument("--module", default="v3.scripts.main.run_cv") + p_s.add_argument("--output-dir", default="v3/results") + p_s.add_argument("--priority", type=int, default=0) + p_s.add_argument("run_args", nargs=argparse.REMAINDER, + help="Args after '--' are forwarded to the module") + + # submit-cv + p_cv = sub.add_parser("submit-cv", + help="Submit one job per rep (distributed 10x5 etc.)") + p_cv.add_argument("--run-name", required=True) + p_cv.add_argument("--reps", type=int, required=True) + p_cv.add_argument("--rep-seed-start", type=int, default=100) + p_cv.add_argument("--rep-seed-step", type=int, default=100) + p_cv.add_argument("--module", default="v3.scripts.main.run_cv") + p_cv.add_argument("--output-dir", default="v3/results") + p_cv.add_argument("--priority", type=int, default=0) + p_cv.add_argument("run_args", nargs=argparse.REMAINDER, + help="Args after '--' forwarded to run_cv (omit --reps/--rep-seed-*)") + + # submit-batch + p_b = sub.add_parser("submit-batch", help="Submit jobs from a JSON file") + p_b.add_argument("batch_file") + + # cancel + p_c = sub.add_parser("cancel", help="Cancel a pending job") + p_c.add_argument("job_id") + + # clear + p_cl = sub.add_parser("clear", help="Delete jobs by run-name, state, or everything") + p_cl.add_argument("--run-name", default=None, help="Delete all jobs with this run-name") + p_cl.add_argument("--states", nargs="+", + default=None, + choices=["done", "failed", "cancelled", "pending", "running"], + help="Delete jobs in these states (default: done+failed+cancelled)") + p_cl.add_argument("--all", action="store_true", help="Delete ALL jobs") + + args = ap.parse_args() + + if not args.server: + ap.error("--server is required (or set HT_SERVER)") + if not args.token: + ap.error("--token is required (or set HT_TOKEN)") + + # strip leading "--" from run_args if present + if hasattr(args, "run_args") and args.run_args and args.run_args[0] == "--": + args.run_args = args.run_args[1:] + + api = _API(args.server, args.token) + + dispatch = { + "clients": cmd_clients, + "jobs": cmd_jobs, + "submit": cmd_submit, + "submit-cv": cmd_submit_cv, + "submit-batch": cmd_submit_batch, + "cancel": cmd_cancel, + "clear": cmd_clear, + } + dispatch[args.cmd](api, args) + + +if __name__ == "__main__": + main() diff --git a/v3/distributed/client.py b/v3/distributed/client.py new file mode 100644 index 0000000..8cf593a --- /dev/null +++ b/v3/distributed/client.py @@ -0,0 +1,435 @@ +""" +HyperTower distributed job client daemon. + +Registers with the server, polls for jobs, syncs code, runs training, +uploads results, and loops. Parses stdout to stream live status. + +Usage: + python -m v3.distributed.client \\ + --server http://apollo:8765 \\ + --token \\ + --server-ssh rpotter@apollo \\ + --server-path /home/rpotter/hypertower \\ + [--local-path ~/hypertower] \\ + [--poll-interval 15] + +Compatibility test (verify GPU env, 1 epoch dry-run): + python -m v3.distributed.client ... --test [extra run_cv args] +""" +from __future__ import annotations + +import argparse +import os +import re +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import threading +import time +from pathlib import Path +from typing import Optional + +import requests + +from .protocol import ( + JobResult, + JobSpec, + PollResponse, + RegisterRequest, + RegisterResponse, + StatusPush, +) + +# ────────────────────────────────────────────────────────────── +# Server HTTP wrapper +# ────────────────────────────────────────────────────────────── + +class _Server: + def __init__(self, base_url: str, token: str): + self.base_url = base_url.rstrip("/") + self._h = {"x-token": token} + self.client_id: str = "" + + def _post(self, path: str, **kw) -> dict: + r = requests.post(f"{self.base_url}{path}", headers=self._h, timeout=15, **kw) + r.raise_for_status() + return r.json() + + def register(self, hostname: str, gpu_info: str) -> str: + data = self._post("/register", + json={"hostname": hostname, "gpu_info": gpu_info}) + self.client_id = data["client_id"] + self.hostname = hostname + self.gpu_info = gpu_info + return self.client_id + + def _reregister(self): + try: + self._post("/register", + json={"hostname": self.hostname, "gpu_info": self.gpu_info}, + params={"reuse_id": self.client_id}) + except Exception: + pass + + def poll(self) -> Optional[JobSpec]: + data = self._post("/poll", params={"client_id": self.client_id}) + if data.get("please_reregister"): + self._reregister() + return JobSpec(**data["job"]) if data.get("job") else None + + def push_status(self, status: StatusPush): + try: + r = requests.post( + f"{self.base_url}/status/{self.client_id}", + json=status.model_dump(), + headers=self._h, + timeout=5, + ) + if r.ok and r.json().get("please_reregister"): + self._reregister() + except Exception: + pass # don't crash job on status push failure + + def complete(self, job_id: str, success: bool, error_msg: Optional[str] = None): + self._post("/complete", + json={"job_id": job_id, "success": success, "error_msg": error_msg}) + + +# ────────────────────────────────────────────────────────────── +# GPU info +# ────────────────────────────────────────────────────────────── + +def _gpu_info() -> str: + # NVIDIA + try: + out = subprocess.check_output( + ["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader"], + text=True, stderr=subprocess.DEVNULL, + ).strip() + if out: + return " | ".join(out.splitlines()) + except Exception: + pass + # AMD + try: + out = subprocess.check_output( + ["rocm-smi", "--showproductname", "--csv"], + text=True, stderr=subprocess.DEVNULL, + ).strip().splitlines() + names = [l for l in out if l and not l.startswith("device")] + if names: + return "AMD: " + " | ".join(names) + except Exception: + pass + # AMD fallback + try: + out = subprocess.check_output( + ["rocminfo"], + text=True, stderr=subprocess.DEVNULL, + ) + names = [l.split(":", 1)[1].strip() for l in out.splitlines() + if "Marketing Name:" in l] + if names: + return "AMD: " + " | ".join(names) + except Exception: + pass + return "no-gpu" + + +# ────────────────────────────────────────────────────────────── +# rsync helpers +# ────────────────────────────────────────────────────────────── + +def _rsync(src: str, dst: str, delete: bool = False): + cmd = ["rsync", "-az", "--info=progress2"] + if delete: + cmd.append("--delete") + cmd += [src, dst] + subprocess.run(cmd, check=True) + + +def _sync_code(server_ssh: str, server_path: str, local_path: str): + """Pull v3/ source from server → local (overwrites local changes).""" + src = f"{server_ssh}:{server_path}/v3/" + dst = f"{local_path}/v3/" + Path(dst).mkdir(parents=True, exist_ok=True) + _rsync(src, dst, delete=True) + + +def _upload_results(server_ssh: str, server_path: str, local_path: str, + run_name: str, output_dir: str): + src = f"{local_path}/{output_dir}/{run_name}/" + dst = f"{server_ssh}:{server_path}/{output_dir}/{run_name}/" + # Ensure parent directory exists on server before rsyncing + remote_parent = f"{server_path}/{output_dir}/{Path(run_name).parent}" + subprocess.run(["ssh", server_ssh, f"mkdir -p '{remote_parent}'"], check=True) + _rsync(src, dst) + + +def _clean_local(local_path: str, run_name: str, output_dir: str): + target = Path(local_path) / output_dir / run_name + if target.exists(): + shutil.rmtree(target) + print(f"[client] cleaned {target}", flush=True) + + +# ────────────────────────────────────────────────────────────── +# Stdout parsers (match v3_hypertower.py print format) +# ────────────────────────────────────────────────────────────── + +# " ep 3/ 30 (6.5s) auc=0.75 acc=0.82" +_EP_RE = re.compile( + r"ep\s+(\d+)/\s*(\d+)\s+\(([0-9.]+)s\)\s+auc=([0-9.nan]+)\s+acc=([0-9.nan]+)" +) +# "[binary:single] fold 2/5" +_FOLD_RE = re.compile(r"fold\s+(\d+)/\d+", re.IGNORECASE) +# "Rep 3 fold_seed=..." or "Rep 3/10 fold_seed=..." +_REP_RE = re.compile(r"Rep\s+(\d+)", re.IGNORECASE) + + +def _parse_line(line: str) -> dict: + """Return any structured fields found in a stdout line.""" + out = {} + m = _FOLD_RE.search(line) + if m: + out["fold"] = int(m.group(1)) + m = _REP_RE.search(line) + if m: + out["rep"] = int(m.group(1)) + m = _EP_RE.search(line) + if m: + out["epoch"] = int(m.group(1)) + out["total_epochs"] = int(m.group(2)) + out["last_epoch_secs"] = float(m.group(3)) + try: + out["last_auc"] = float(m.group(4)) + except ValueError: + pass + return out + + +# ────────────────────────────────────────────────────────────── +# Core job runner +# ────────────────────────────────────────────────────────────── + +def _run_job(job: JobSpec, server: _Server, + server_ssh: str, server_path: str, local_path: str, + no_sync: bool = False, num_workers: Optional[int] = None, + extra_args: list[str] | None = None) -> bool: + extra_args = extra_args or [] + # 1. Sync code + if no_sync: + print(f"[client] skipping sync (--no-sync)", flush=True) + else: + print(f"[client] syncing v3/ from server...", flush=True) + server.push_status(StatusPush(state="syncing", job_id=job.job_id, run_name=job.run_name)) + _sync_code(server_ssh, server_path, local_path) + + # 2. Launch training subprocess + job_args = job.args + if num_workers is not None and "--num-workers" not in job_args: + job_args = job_args + ["--num-workers", str(num_workers)] + cmd = [sys.executable, "-m", job.module] + job_args + extra_args + print(f"[client] running: {' '.join(cmd)}", flush=True) + server.push_status(StatusPush(state="running", job_id=job.job_id, run_name=job.run_name)) + + # Write stdout to a temp file so the child's fd table is unmodified — + # this lets Python 3.14's forkserver communicate over its own pipes without + # interference. We tail the file from a thread for live status. + log_dir = Path(local_path) / "v3" / "distributed" / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + log_file = log_dir / f"job_{job.job_id}.log" + ctx: dict = {} + + def _tail(path: Path): + with open(path, "r") as f: + while True: + raw = f.readline() + if raw: + print(raw, end="", flush=True) + info = _parse_line(raw) + ctx.update(info) + if "epoch" in info: + server.push_status(StatusPush( + state="running", + job_id=job.job_id, + run_name=job.run_name, + rep=ctx.get("rep"), + fold=ctx.get("fold"), + epoch=ctx.get("epoch"), + total_epochs=ctx.get("total_epochs"), + last_epoch_secs=ctx.get("last_epoch_secs"), + last_auc=ctx.get("last_auc"), + )) + elif proc.poll() is not None: + # Drain any remaining output then exit + for raw in f: + print(raw, end="", flush=True) + break + else: + time.sleep(0.05) + + with open(log_file, "w") as logf: + proc = subprocess.Popen( + cmd, + stdout=logf, + stderr=logf, + cwd=local_path, + start_new_session=True, # own process group so we can kill all workers on failure + ) + + tailer = threading.Thread(target=_tail, args=(log_file,), daemon=True) + tailer.start() + proc.wait() + tailer.join(timeout=5) + log_file.unlink(missing_ok=True) + + success = proc.returncode == 0 + + if not success: + # Kill any surviving worker processes in the group (e.g. DataLoader workers + # that outlived the main process after an OOM kill) + try: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + except ProcessLookupError: + pass # already gone + + if not success: + print(f"[client] job FAILED (rc={proc.returncode})", flush=True) + return False + + # 3. Upload results (skip when client IS the server — results already in place) + if no_sync: + print(f"[client] skipping upload (--no-sync, results already local)", flush=True) + else: + print(f"[client] uploading results...", flush=True) + server.push_status(StatusPush(state="uploading", job_id=job.job_id, run_name=job.run_name)) + _upload_results(server_ssh, server_path, local_path, job.run_name, job.output_dir) + + # 4. Clean local (only remote clients need cleanup) + _clean_local(local_path, job.run_name, job.output_dir) + print(f"[client] job {job.job_id} complete.", flush=True) + return True + + +# ────────────────────────────────────────────────────────────── +# Compatibility test +# ────────────────────────────────────────────────────────────── + +def _compat_test(server: _Server, server_ssh: str, server_path: str, + local_path: str, extra_args: list[str], no_sync: bool = False): + """Run 1 epoch on 1 rep to verify the env works end-to-end.""" + print("[client] === compatibility test ===", flush=True) + job = JobSpec( + job_id="compat-test", + run_name="_compat_test", + module="v3.scripts.main.run_cv", + args=[ + "--bridge-mode", "image_only", + "--backbone", "resnet50", + "--epochs", "1", + "--reps", "1", + "--rep-seed-start", "42", + "--in-memory-cache", + "--num-workers", "0", + "--output-root", "v3/results", + "--run-name", "_compat_test", + ] + extra_args, + output_dir="v3/results", + ) + ok = _run_job(job, server, server_ssh, server_path, local_path, + no_sync=no_sync, extra_args=[a for a in extra_args if a != "--"]) + # always clean up test results + _clean_local(local_path, "_compat_test", "v3/results") + if ok: + print("[client] compatibility test PASSED ✓", flush=True) + else: + print("[client] compatibility test FAILED ✗", flush=True) + return ok + + +# ────────────────────────────────────────────────────────────── +# Main daemon +# ────────────────────────────────────────────────────────────── + +def main(): + ap = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + ap.add_argument("--server", required=True, + help="Server URL, e.g. http://apollo:8765") + ap.add_argument("--token", default=os.environ.get("HT_TOKEN", ""), + help="Shared secret (or set HT_TOKEN env var)") + ap.add_argument("--server-ssh", required=True, + help="SSH target for rsync, e.g. rpotter@apollo") + ap.add_argument("--server-path", required=True, + help="Absolute path to hypertower root on server") + ap.add_argument("--local-path", + default=str(Path.home() / "hypertower"), + help="Absolute path to hypertower root on this machine") + ap.add_argument("--poll-interval", type=int, default=15, + help="Seconds to wait between polls when idle") + ap.add_argument("--num-workers", type=int, default=None, + help="Override DataLoader num_workers for all jobs on this client.") + ap.add_argument("--extra-args", nargs=argparse.REMAINDER, default=[], + help="Extra args appended to every job on this client (e.g. --cache-workers 0). " + "Use -- to separate from client args: --extra-args -- --cache-workers 0") + ap.add_argument("--no-sync", action="store_true", + help="Skip rsync of v3/ before each job (use when client IS the server)") + ap.add_argument("--test", action="store_true", + help="Run 1-epoch compatibility test and exit") + ap.add_argument("extra_args", nargs="*", + help="Extra args forwarded to run_cv in --test mode") + args = ap.parse_args() + + if not args.token: + ap.error("--token is required (or set HT_TOKEN)") + + hostname = socket.gethostname() + gpu_info = _gpu_info() + server = _Server(args.server, args.token) + + client_id = server.register(hostname, gpu_info) + print(f"[client] registered as {client_id} ({hostname} | {gpu_info})", flush=True) + + if args.test: + sys.exit(0 if _compat_test( + server, args.server_ssh, args.server_path, + args.local_path, args.extra_args, + no_sync=args.no_sync, + ) else 1) + + print(f"[client] polling every {args.poll_interval}s...", flush=True) + while True: + try: + job = server.poll() + if job is None: + server.push_status(StatusPush(state="idle")) + time.sleep(args.poll_interval) + continue + + success = _run_job( + job, server, + args.server_ssh, args.server_path, args.local_path, + no_sync=args.no_sync, + num_workers=args.num_workers, + extra_args=[a for a in args.extra_args if a != "--"], + ) + server.complete(job.job_id, success, + error_msg=None if success else "non-zero exit code") + server.push_status(StatusPush(state="idle")) + + except KeyboardInterrupt: + print("\n[client] shutting down", flush=True) + break + except Exception as exc: + print(f"[client] error: {exc}", flush=True) + time.sleep(args.poll_interval) + + +if __name__ == "__main__": + main() diff --git a/v3/distributed/jobs.db b/v3/distributed/jobs.db new file mode 100644 index 0000000..e96b06f Binary files /dev/null and b/v3/distributed/jobs.db differ diff --git a/v3/distributed/protocol.py b/v3/distributed/protocol.py new file mode 100644 index 0000000..f1ed195 --- /dev/null +++ b/v3/distributed/protocol.py @@ -0,0 +1,62 @@ +"""Shared data models for server/client communication.""" +from __future__ import annotations + +from typing import Optional +from pydantic import BaseModel + + +class RegisterRequest(BaseModel): + hostname: str + gpu_info: str + + +class RegisterResponse(BaseModel): + client_id: str + + +class StatusPush(BaseModel): + state: str # idle | syncing | running | uploading | error + job_id: Optional[str] = None + run_name: Optional[str] = None + rep: Optional[int] = None + fold: Optional[int] = None + epoch: Optional[int] = None + total_epochs: Optional[int] = None + last_epoch_secs: Optional[float] = None + last_auc: Optional[float] = None + error: Optional[str] = None + + +class ClientInfo(BaseModel): + client_id: str + hostname: str + gpu_info: str + status: StatusPush + last_seen: str + + +class JobSpec(BaseModel): + job_id: str + run_name: str + module: str # e.g. "v3.scripts.main.run_cv" + args: list[str] + output_dir: str = "v3/results" # local subdir for rsync-back + + +class PollResponse(BaseModel): + job: Optional[JobSpec] = None + please_reregister: bool = False + + +class JobResult(BaseModel): + job_id: str + success: bool + error_msg: Optional[str] = None + + +class JobSubmit(BaseModel): + run_name: str + module: str = "v3.scripts.main.run_cv" + args: list[str] + output_dir: str = "v3/results" + priority: int = 0 diff --git a/v3/distributed/server.py b/v3/distributed/server.py new file mode 100644 index 0000000..0e324fc --- /dev/null +++ b/v3/distributed/server.py @@ -0,0 +1,402 @@ +""" +HyperTower distributed job server. + +Manages a SQLite job queue and a registry of connected clients. +Clients poll for work, push status updates, and report completion. + +Usage: + python -m v3.distributed.server --port 8765 --token + +Environment: + HT_TOKEN — fallback if --token is not passed +""" +from __future__ import annotations + +import argparse +import json +import os +import sqlite3 +import threading +import time +import uuid +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +from fastapi import Depends, FastAPI, Header, HTTPException +import uvicorn + +from .protocol import ( + ClientInfo, + JobResult, + JobSpec, + JobSubmit, + PollResponse, + RegisterRequest, + RegisterResponse, + StatusPush, +) + +# ────────────────────────────────────────────────────────────── +# Global state +# ────────────────────────────────────────────────────────────── + +_TOKEN: str = "" +_DB_PATH: Path = Path("v3/distributed/jobs.db") +_CLIENT_TTL: int = 120 # seconds before a client is considered gone +_MAX_ATTEMPTS: int = 3 # max times a job is retried before being left as failed + +_clients: dict[str, ClientInfo] = {} +_clients_lock = threading.Lock() + + +def _reap_stale_clients(): + """Background thread: remove silent clients and re-queue their running jobs.""" + while True: + time.sleep(30) + cutoff = datetime.now(timezone.utc).timestamp() - _CLIENT_TTL + + # Step 1: evict timed-out clients from registry + with _clients_lock: + stale = [ + cid for cid, c in _clients.items() + if datetime.fromisoformat(c.last_seen).timestamp() < cutoff + ] + for cid in stale: + print(f"[server] reaped stale client {cid} ({_clients[cid].hostname})", flush=True) + del _clients[cid] + known_ids = set(_clients.keys()) + + # Step 2: reset any running job whose assigned client is no longer known + with _db() as conn: + rows = conn.execute( + "SELECT job_id, assigned_to FROM jobs WHERE state='running'" + ).fetchall() + for row in rows: + if row["assigned_to"] not in known_ids: + conn.execute( + "UPDATE jobs SET state='pending', assigned_to=NULL, started_at=NULL " + "WHERE job_id=?", + (row["job_id"],) + ) + print(f"[server] re-queued job {row['job_id']} " + f"(client {row['assigned_to']} unknown)", flush=True) + +# ────────────────────────────────────────────────────────────── +# Database helpers +# ────────────────────────────────────────────────────────────── + +@contextmanager +def _db(): + conn = sqlite3.connect(str(_DB_PATH)) + conn.row_factory = sqlite3.Row + try: + yield conn + conn.commit() + finally: + conn.close() + + +def _init_db(): + _DB_PATH.parent.mkdir(parents=True, exist_ok=True) + with _db() as conn: + conn.execute(""" + CREATE TABLE IF NOT EXISTS jobs ( + job_id TEXT PRIMARY KEY, + run_name TEXT NOT NULL, + module TEXT NOT NULL, + args TEXT NOT NULL, -- JSON list + output_dir TEXT NOT NULL DEFAULT 'v3/results', + state TEXT NOT NULL DEFAULT 'pending', + priority INTEGER NOT NULL DEFAULT 0, + assigned_to TEXT, + created_at TEXT NOT NULL, + started_at TEXT, + completed_at TEXT, + error_msg TEXT, + attempts INTEGER NOT NULL DEFAULT 0 + ) + """) + # Add attempts column to existing DBs that predate this field + try: + conn.execute("ALTER TABLE jobs ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0") + except Exception: + pass # column already exists + # Note: running jobs are NOT reset on startup — active clients will re-register + # via poll/status and the reaper will clean up any that don't reconnect within TTL. + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _ensure_client(client_id: str, hostname: str = "", gpu_info: str = "") -> bool: + """Re-register a client that survived a server restart. + Returns True if the client was unknown (placeholder created) so the caller + can ask the client to re-register with full info.""" + if client_id not in _clients: + _clients[client_id] = ClientInfo( + client_id=client_id, + hostname=hostname or client_id, + gpu_info=gpu_info or "unknown", + status=StatusPush(state="idle"), + last_seen=_now(), + ) + print(f"[server] re-registered {client_id} (survived restart)", flush=True) + return True + return False + +# ────────────────────────────────────────────────────────────── +# FastAPI app +# ────────────────────────────────────────────────────────────── + +app = FastAPI(title="HyperTower Job Server") + + +def _check_token(x_token: str = Header(...)): + if x_token != _TOKEN: + raise HTTPException(status_code=403, detail="Invalid token") + + +# ── Registration ────────────────────────────────────────────── + +@app.post("/register", response_model=RegisterResponse, + dependencies=[Depends(_check_token)]) +def register(req: RegisterRequest, reuse_id: Optional[str] = None): + with _clients_lock: + client_id = reuse_id if (reuse_id and reuse_id in _clients) else str(uuid.uuid4())[:8] + existing_status = _clients[client_id].status if client_id in _clients else StatusPush(state="idle") + _clients[client_id] = ClientInfo( + client_id=client_id, + hostname=req.hostname, + gpu_info=req.gpu_info, + status=existing_status, + last_seen=_now(), + ) + action = "re-registered" if reuse_id else "registered" + print(f"[server] {action} {client_id} ({req.hostname} | {req.gpu_info})", flush=True) + return RegisterResponse(client_id=client_id) + + +# ── Job polling ─────────────────────────────────────────────── + +@app.post("/poll", response_model=PollResponse, + dependencies=[Depends(_check_token)]) +def poll(client_id: str): + with _clients_lock: + needs_reregister = _ensure_client(client_id) + _clients[client_id].last_seen = _now() + + with _db() as conn: + row = conn.execute( + "SELECT * FROM jobs WHERE state='pending' " + "ORDER BY priority DESC, created_at ASC LIMIT 1" + ).fetchone() + + if row is None: + return PollResponse(job=None, please_reregister=needs_reregister) + + job_id = row["job_id"] + # Reset any other running jobs for this client — client can only work on one at a time. + # This cleans up orphans left over from server restarts. + cur = conn.execute( + "UPDATE jobs SET state='pending', assigned_to=NULL, started_at=NULL " + "WHERE assigned_to=? AND state='running' AND job_id!=?", + (client_id, job_id), + ) + if cur.rowcount: + print(f"[server] reset {cur.rowcount} orphaned running job(s) for {client_id}", flush=True) + conn.execute( + "UPDATE jobs SET state='running', assigned_to=?, started_at=? WHERE job_id=?", + (client_id, _now(), job_id), + ) + + job = JobSpec( + job_id=job_id, + run_name=row["run_name"], + module=row["module"], + args=json.loads(row["args"]), + output_dir=row["output_dir"], + ) + + with _clients_lock: + _clients[client_id].status = StatusPush( + state="syncing", job_id=job_id, run_name=row["run_name"] + ) + + print(f"[server] dispatched {job_id} ({row['run_name']}) → {client_id}", flush=True) + return PollResponse(job=job, please_reregister=needs_reregister) + + +# ── Status ──────────────────────────────────────────────────── + +@app.post("/status/{client_id}", dependencies=[Depends(_check_token)]) +def push_status(client_id: str, status: StatusPush): + with _clients_lock: + needs_reregister = _ensure_client(client_id) + _clients[client_id].status = status + _clients[client_id].last_seen = _now() + return {"ok": True, "please_reregister": needs_reregister} + + +@app.get("/clients", dependencies=[Depends(_check_token)]) +def list_clients(): + with _clients_lock: + return list(_clients.values()) + + +@app.get("/clients/{client_id}", dependencies=[Depends(_check_token)]) +def get_client(client_id: str): + with _clients_lock: + if client_id not in _clients: + raise HTTPException(status_code=404, detail="Unknown client") + return _clients[client_id] + + +# ── Job completion ──────────────────────────────────────────── + +@app.post("/complete", dependencies=[Depends(_check_token)]) +def complete(result: JobResult): + with _db() as conn: + if result.success: + conn.execute( + "UPDATE jobs SET state='done', completed_at=?, error_msg=NULL WHERE job_id=?", + (_now(), result.job_id), + ) + print(f"[server] job {result.job_id} → done", flush=True) + + # Auto-clear run if all jobs for this run_name are now done + run_row = conn.execute( + "SELECT run_name FROM jobs WHERE job_id=?", (result.job_id,) + ).fetchone() + if run_row: + run_name = run_row["run_name"] + remaining = conn.execute( + "SELECT COUNT(*) FROM jobs WHERE run_name=? AND state != 'done'", + (run_name,) + ).fetchone()[0] + if remaining == 0: + total = conn.execute( + "SELECT COUNT(*) FROM jobs WHERE run_name=?", (run_name,) + ).fetchone()[0] + conn.execute("DELETE FROM jobs WHERE run_name=?", (run_name,)) + print(f"[server] run '{run_name}' complete ({total} jobs) — cleared", flush=True) + else: + row = conn.execute( + "SELECT attempts FROM jobs WHERE job_id=?", (result.job_id,) + ).fetchone() + attempts = (row["attempts"] if row else 0) + 1 + if attempts < _MAX_ATTEMPTS: + conn.execute( + "UPDATE jobs SET state='pending', assigned_to=NULL, started_at=NULL, " + "attempts=?, error_msg=? WHERE job_id=?", + (attempts, result.error_msg, result.job_id), + ) + print(f"[server] job {result.job_id} failed (attempt {attempts}/{_MAX_ATTEMPTS}), " + f"re-queuing", flush=True) + else: + conn.execute( + "UPDATE jobs SET state='failed', completed_at=?, attempts=?, error_msg=? " + "WHERE job_id=?", + (_now(), attempts, result.error_msg, result.job_id), + ) + print(f"[server] job {result.job_id} failed permanently after " + f"{attempts} attempts", flush=True) + return {"ok": True} + + +# ── Job queue management ────────────────────────────────────── + +@app.post("/jobs", dependencies=[Depends(_check_token)]) +def submit_job(job: JobSubmit): + job_id = str(uuid.uuid4())[:12] + with _db() as conn: + conn.execute( + "INSERT INTO jobs " + "(job_id, run_name, module, args, output_dir, priority, created_at) " + "VALUES (?,?,?,?,?,?,?)", + (job_id, job.run_name, job.module, json.dumps(job.args), + job.output_dir, job.priority, _now()), + ) + print(f"[server] queued {job_id} ({job.run_name})", flush=True) + return {"job_id": job_id} + + +@app.get("/jobs", dependencies=[Depends(_check_token)]) +def list_jobs(state: Optional[str] = None): + with _db() as conn: + if state: + rows = conn.execute( + "SELECT * FROM jobs WHERE state=? ORDER BY created_at DESC", (state,) + ).fetchall() + else: + rows = conn.execute( + "SELECT * FROM jobs ORDER BY created_at DESC" + ).fetchall() + return [dict(r) for r in rows] + + +@app.post("/jobs/clear", dependencies=[Depends(_check_token)]) +def clear_jobs(body: dict): + with _db() as conn: + if body.get("all"): + cur = conn.execute("DELETE FROM jobs") + elif body.get("run_name"): + cur = conn.execute("DELETE FROM jobs WHERE run_name=?", (body["run_name"],)) + else: + states = body.get("states", ["done", "failed", "cancelled"]) + placeholders = ",".join("?" * len(states)) + cur = conn.execute(f"DELETE FROM jobs WHERE state IN ({placeholders})", states) + print(f"[server] cleared {cur.rowcount} jobs", flush=True) + return {"cleared": cur.rowcount} + + +@app.delete("/jobs/{job_id}", dependencies=[Depends(_check_token)]) +def cancel_job(job_id: str): + with _db() as conn: + conn.execute( + "UPDATE jobs SET state='cancelled' WHERE job_id=? AND state='pending'", + (job_id,), + ) + return {"ok": True} + + +# ────────────────────────────────────────────────────────────── +# Entry point +# ────────────────────────────────────────────────────────────── + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--port", type=int, default=8765) + ap.add_argument("--host", default="0.0.0.0") + ap.add_argument("--token", default=os.environ.get("HT_TOKEN", ""), + help="Shared secret (or set HT_TOKEN env var)") + ap.add_argument("--db", default="v3/distributed/jobs.db", + help="Path to SQLite job database") + ap.add_argument("--client-ttl", type=int, default=120, + help="Seconds of silence before a client is reaped (default: 120)") + ap.add_argument("--max-attempts", type=int, default=3, + help="Max times a failed job is retried before being left as failed (default: 3)") + args = ap.parse_args() + + if not args.token: + ap.error("--token is required (or set HT_TOKEN)") + + global _TOKEN, _DB_PATH, _CLIENT_TTL, _MAX_ATTEMPTS + _TOKEN = args.token + _DB_PATH = Path(args.db) + _CLIENT_TTL = args.client_ttl + _MAX_ATTEMPTS = args.max_attempts + _init_db() + + reaper = threading.Thread(target=_reap_stale_clients, daemon=True) + reaper.start() + + print(f"[server] listening on {args.host}:{args.port} client_ttl={_CLIENT_TTL}s", flush=True) + uvicorn.run(app, host=args.host, port=args.port, log_level="warning") + + +if __name__ == "__main__": + main() diff --git a/v3/distributed/server_cheat_sheet.md b/v3/distributed/server_cheat_sheet.md new file mode 100644 index 0000000..67f86e3 --- /dev/null +++ b/v3/distributed/server_cheat_sheet.md @@ -0,0 +1,122 @@ +# Distributed Server Cheat Sheet + +All commands assume server is running on hades at port 8765. + +## Start Server +```bash +python -m v3.distributed.server --token hypertower +``` +Run inside tmux so it survives disconnects: +```bash +tmux new -s htserver +python -m v3.distributed.server --token hypertower +# Ctrl-B D to detach +tmux attach -t htserver # reattach later +``` + +## Start Clients + +**Hades (server-local, no sync):** +```bash +python -m v3.distributed.client \ + --server http://hades:8765 --token hypertower \ + --server-ssh ignored --server-path ignored \ + --local-path /home/rpotter/hypertower \ + --no-sync +``` + +**Apollo (remote client):** +```bash +python -m v3.distributed.client \ + --server http://hades:8765 --token hypertower \ + --server-ssh rpotter@hades \ + --server-path /home/rpotter/hypertower \ + --local-path /home/odin/hypertower +``` + +--- + +## Monitoring + +**Live client monitor (refreshes every 5s):** +```bash +python -m v3.distributed.cli --server http://hades:8765 --token hypertower clients --watch +``` + +**Faster refresh:** +```bash +python -m v3.distributed.cli --server http://hades:8765 --token hypertower clients --watch --interval 2 +``` + +**Inspect a single client:** +```bash +python -m v3.distributed.cli --server http://hades:8765 --token hypertower clients +``` + +**View job queue:** +```bash +python -m v3.distributed.cli --server http://hades:8765 --token hypertower jobs +``` + +**Filter by state:** +```bash +python -m v3.distributed.cli --server http://hades:8765 --token hypertower jobs --state pending +python -m v3.distributed.cli --server http://hades:8765 --token hypertower jobs --state running +python -m v3.distributed.cli --server http://hades:8765 --token hypertower jobs --state failed +``` + +--- + +## Submitting Jobs + +**Phase 3 main grid:** +```bash +python -m v3.scripts.main.phase3.dispatch_phase3 \ + --server http://hades:8765 --token hypertower +``` + +**Dry run (check what would be submitted):** +```bash +python -m v3.scripts.main.phase3.dispatch_phase3 \ + --server http://hades:8765 --token hypertower --dry-run +``` + +**Other grids:** +```bash +python -m v3.scripts.main.phase3.dispatch_phase3 \ + --server http://hades:8765 --token hypertower \ + --grid v3/scripts/main/phase3/epoch_grid.json + +python -m v3.scripts.main.phase3.dispatch_phase3 \ + --server http://hades:8765 --token hypertower \ + --grid v3/scripts/main/phase3/phase35_grid.json +``` + +--- + +## Queue Management + +**Clear failed jobs:** +```bash +python -m v3.distributed.cli --server http://hades:8765 --token hypertower clear --states failed +``` + +**Clear running jobs (orphan cleanup):** +```bash +python -m v3.distributed.cli --server http://hades:8765 --token hypertower clear --states running +``` + +**Clear all jobs:** +```bash +python -m v3.distributed.cli --server http://hades:8765 --token hypertower clear --all +``` + +**Clear by run name:** +```bash +python -m v3.distributed.cli --server http://hades:8765 --token hypertower clear --run-name phase3/baseline +``` + +**Cancel a specific job:** +```bash +python -m v3.distributed.cli --server http://hades:8765 --token hypertower cancel +``` diff --git a/v3/docs/phase_plan.md b/v3/docs/phase_plan.md new file mode 100644 index 0000000..f86bf01 --- /dev/null +++ b/v3/docs/phase_plan.md @@ -0,0 +1,81 @@ +# HyperTower Phase Plan + +## Phase 1 — PAPILA Baseline Reproduction +**Goal:** Reproduce the CNN results reported in the PAPILA paper and establish proper evaluation methodology. + +**Covers:** +- Reproduce PAPILA paper AUC results for VGG16, MobileNetV2, ResNet50, DenseNet121, InceptionV3 +- Present results with 95% CI and boxplot across repeated CV folds +- **Show effect of data leakage:** compare leaky CV (same patient in train and test) vs proper patient-stratified CV — motivates repeated CV methodology + +**Status:** Complete + +--- + +## Phase 2 — Image Preprocessing & Backbone Selection +**Goal:** Show the effect of image-level design choices on classification performance. + +**Covers:** +- **Show the effect of pre-training:** Refugelike (REFUGE-pretrained) backbone vs standard ImageNet backbones +- **Show the effect of cropping:** U-Net optic disc crop vs no cropping, and crop scale sensitivity + +**Status:** Complete — best config is refugelike backbone, no crop (proper CV) + +--- + +## Phase 3 — Clinical Data Fusion (Single-Eye) +**Goal:** Show the effect of combining fundus image features with clinical metadata in the single-eye pipeline. + +**Covers:** +- **Show the effect of combining image data with clinical data:** fused bridge vs image-only and clinical-only ablations +- Architecture search: loss function (BCD vs all-losses), SE attention, IOP correction strategy, feature ablations, network dimensions, dropout, learning rate, warmup strategy, augmentation, balanced sampling +- Epoch length sensitivity + +**Status:** Complete — key findings: IOP ratio correction + drop raw (+1.7%), excluding axial length helps, age is most informative clinical feature, SE adds no benefit, LR very sensitive + +--- + +## Phase 3.5 — Confirmation & Tuning +**Goal:** Confirm that the top phase 3 findings combine additively, and tune BCD probability with the best IOP preprocessing. + +**Covers:** +- Combine `iop_ratio_drop_raw` (best preprocessing) with `bcd_p07` (best loss setting) +- Extend BCD probability sweep to p=0.8 and p=0.9 to find the optimum + +**Planned approach:** +- Baseline is phase 3 `iop_ratio_drop_raw` (0.8685 ± 0.011) +- All runs use best single-eye settings: refugelike, no crop, ratio IOP + drop raw, no axial length + +**Status:** Not started + +--- + +## Phase 4 — Dual CNN Architecture (Image Only) +**Goal:** Show the effect of processing both eyes jointly, and compare bilateral architectures against the single-eye baseline. + +**Covers:** +- **Show the effect of a dual CNN:** bilateral tower (both eyes) vs single-eye tower, image data only +- **Architecture comparison:** canonical HyperTower bilateral (siamese shared-weight backbone returning mean+delta features) vs independent per-eye processing with late fusion + +**Planned approach:** +- Image-only, no clinical data — isolates the bilateral vision question cleanly +- Use best image settings from phase 2 (refugelike, no crop) +- Compare siamese tower, independent bilateral, and single-eye (phase 3 baseline) directly + +**Status:** Not started + +--- + +## Phase 5 — Full HyperTower: Bilateral + Clinical Data + Fusion Heads +**Goal:** Bring together the best bilateral architecture (phase 4) with clinical data fusion (phase 3), and compare prediction aggregation strategies. + +**Covers:** +- **Show the effect of a dual CNN + clinical data:** bilateral tower with fused clinical bridge — the full HyperTower model +- **Ensemble vs fused head:** patient-level prediction via ensemble (average OD+OS eye-level scores) vs learned fused head trained on top of single-eye scores + +**Planned approach:** +- Use best settings from all prior phases (refugelike, no crop, ratio IOP + drop raw, no axial length) +- Compare tower modes: single, bilateral, ensemble, fused-head +- Establish final best configuration as the HyperTower result + +**Status:** Not started diff --git a/v3/figures/architecture_ensemble.png b/v3/figures/architecture_ensemble.png new file mode 100644 index 0000000..d6c02f1 Binary files /dev/null and b/v3/figures/architecture_ensemble.png differ diff --git a/v3/figures/architecture_fused_head.png b/v3/figures/architecture_fused_head.png new file mode 100644 index 0000000..13c7d00 Binary files /dev/null and b/v3/figures/architecture_fused_head.png differ diff --git a/v3/figures/architecture_hypertower.png b/v3/figures/architecture_hypertower.png new file mode 100644 index 0000000..d108656 Binary files /dev/null and b/v3/figures/architecture_hypertower.png differ diff --git a/v3/figures/architecture_single_tower.png b/v3/figures/architecture_single_tower.png new file mode 100644 index 0000000..93c5fe6 Binary files /dev/null and b/v3/figures/architecture_single_tower.png differ diff --git a/v3/figures/cross_phase_progression.png b/v3/figures/cross_phase_progression.png new file mode 100644 index 0000000..5f881ae Binary files /dev/null and b/v3/figures/cross_phase_progression.png differ diff --git a/v3/figures/explainability/comparison_panel_phase5.png b/v3/figures/explainability/comparison_panel_phase5.png new file mode 100644 index 0000000..67c71db Binary files /dev/null and b/v3/figures/explainability/comparison_panel_phase5.png differ diff --git a/v3/figures/explainability/confidence_strips.png b/v3/figures/explainability/confidence_strips.png new file mode 100644 index 0000000..0f555a9 Binary files /dev/null and b/v3/figures/explainability/confidence_strips.png differ diff --git a/v3/figures/explainability/confidence_strips_comparison.png b/v3/figures/explainability/confidence_strips_comparison.png new file mode 100644 index 0000000..f7faf82 Binary files /dev/null and b/v3/figures/explainability/confidence_strips_comparison.png differ diff --git a/v3/figures/explainability/gradcam/disc_attention_detail.png b/v3/figures/explainability/gradcam/disc_attention_detail.png new file mode 100644 index 0000000..51b4e3a Binary files /dev/null and b/v3/figures/explainability/gradcam/disc_attention_detail.png differ diff --git a/v3/figures/explainability/gradcam/mean_cam_comparison.png b/v3/figures/explainability/gradcam/mean_cam_comparison.png new file mode 100644 index 0000000..41d070a Binary files /dev/null and b/v3/figures/explainability/gradcam/mean_cam_comparison.png differ diff --git a/v3/figures/explainability/gradcam/mean_cam_glaucoma.png b/v3/figures/explainability/gradcam/mean_cam_glaucoma.png new file mode 100644 index 0000000..0719bf9 Binary files /dev/null and b/v3/figures/explainability/gradcam/mean_cam_glaucoma.png differ diff --git a/v3/figures/explainability/gradcam/mean_cam_normal.png b/v3/figures/explainability/gradcam/mean_cam_normal.png new file mode 100644 index 0000000..a57a325 Binary files /dev/null and b/v3/figures/explainability/gradcam/mean_cam_normal.png differ diff --git a/v3/figures/explainability/gradcam/overlay_grid_glaucoma.png b/v3/figures/explainability/gradcam/overlay_grid_glaucoma.png new file mode 100644 index 0000000..08e25c4 Binary files /dev/null and b/v3/figures/explainability/gradcam/overlay_grid_glaucoma.png differ diff --git a/v3/figures/explainability/gradcam/overlay_grid_normal.png b/v3/figures/explainability/gradcam/overlay_grid_normal.png new file mode 100644 index 0000000..e4610ee Binary files /dev/null and b/v3/figures/explainability/gradcam/overlay_grid_normal.png differ diff --git a/v3/figures/explainability/head_comparison.png b/v3/figures/explainability/head_comparison.png new file mode 100644 index 0000000..ecd0846 Binary files /dev/null and b/v3/figures/explainability/head_comparison.png differ diff --git a/v3/figures/explainability/md_importance_phase5.csv b/v3/figures/explainability/md_importance_phase5.csv new file mode 100644 index 0000000..f611c3e --- /dev/null +++ b/v3/figures/explainability/md_importance_phase5.csv @@ -0,0 +1,10 @@ +feature,mean_importance,std_importance +Age,0.02684933048095948,0.018982159670576627 +IOP_corr,0.01402488800132739,0.016679352675009966 +Phakic/Pseudophakic,0.006225936127828556,0.007852635891143775 +Gender,0.0026848811162960276,0.010922597349168684 +eyeID,0.0009305775513090246,0.005399435288001892 +Pachymetry,0.0006101177888381434,0.002786215613365408 +dioptre_2,-0.0005330790563094697,0.001828442063194257 +astigmatism,-0.003371613580988547,0.0019910628774032386 +dioptre_1,-0.004100123269320834,0.0075533314694558405 diff --git a/v3/figures/explainability/md_importance_phase5.png b/v3/figures/explainability/md_importance_phase5.png new file mode 100644 index 0000000..e26b625 Binary files /dev/null and b/v3/figures/explainability/md_importance_phase5.png differ diff --git a/v3/figures/phase1_clinical_classifiers.png b/v3/figures/phase1_clinical_classifiers.png new file mode 100644 index 0000000..870bc63 Binary files /dev/null and b/v3/figures/phase1_clinical_classifiers.png differ diff --git a/v3/figures/phase1_cnn_backbones.png b/v3/figures/phase1_cnn_backbones.png new file mode 100644 index 0000000..01e0c7e Binary files /dev/null and b/v3/figures/phase1_cnn_backbones.png differ diff --git a/v3/figures/phase2_analysis.png b/v3/figures/phase2_analysis.png new file mode 100644 index 0000000..3122807 Binary files /dev/null and b/v3/figures/phase2_analysis.png differ diff --git a/v3/figures/phase3_modality_ablation.png b/v3/figures/phase3_modality_ablation.png new file mode 100644 index 0000000..8acb071 Binary files /dev/null and b/v3/figures/phase3_modality_ablation.png differ diff --git a/v3/figures/phase3_single_mode_ablations.png b/v3/figures/phase3_single_mode_ablations.png new file mode 100644 index 0000000..863877e Binary files /dev/null and b/v3/figures/phase3_single_mode_ablations.png differ diff --git a/v3/figures/phase4_analysis.png b/v3/figures/phase4_analysis.png new file mode 100644 index 0000000..dd6294c Binary files /dev/null and b/v3/figures/phase4_analysis.png differ diff --git a/v3/figures/phase5_analysis.png b/v3/figures/phase5_analysis.png new file mode 100644 index 0000000..422acc1 Binary files /dev/null and b/v3/figures/phase5_analysis.png differ diff --git a/v3/scripts/development/stage_seg_cnn/__init__.py b/v3/scripts/development/stage_seg_cnn/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/v3/scripts/development/stage_seg_cnn/run_seg_cnn.py b/v3/scripts/development/stage_seg_cnn/run_seg_cnn.py new file mode 100644 index 0000000..1d8da23 --- /dev/null +++ b/v3/scripts/development/stage_seg_cnn/run_seg_cnn.py @@ -0,0 +1,595 @@ +#!/usr/bin/env python +""" +Segmentation-map CNN — glaucoma grading from disc/cup label maps. + +Trains a CNN whose input is the combined optic disc / cup segmentation map +(pixel values 0=bg, 1=disc_rim, 2=cup) rather than the original fundus image. +The model must learn structural relationships like cup-to-disc ratio, rim area, +and cup eccentricity directly from the segmentation geometry. + +Segmentation source (--seg-mode): + gt [default] Rasterise expert contour annotations from the PAPILA manifest. + unet Run a trained UNetSegmenter on the raw fundus image. + Requires --unet-weights. + +Cross-validation: + 5-fold stratified group CV, with both eyes of the same patient always in the + same fold (preventing OD/OS leakage). Folds are built from the PAPILA + clinical CSV, then matched to manifest entries by patient ID + eye. + +Usage examples: + # GT masks, default settings + python -m v3.scripts.main.phase_seg_cnn.run_seg_cnn \\ + --image-dir Papila/FundusImages \\ + --clinical-dir Papila/ClinicalData \\ + --manifest manifest.csv + + # U-Net predicted masks, resnet50 backbone + python -m v3.scripts.main.phase_seg_cnn.run_seg_cnn \\ + --image-dir Papila/FundusImages \\ + --clinical-dir Papila/ClinicalData \\ + --manifest manifest.csv \\ + --seg-mode unet --unet-weights models/unet_segmenter/best.pt \\ + --backbone resnet50 + + # Single-channel label map instead of one-hot + python -m v3.scripts.main.phase_seg_cnn.run_seg_cnn \\ + --image-dir Papila/FundusImages \\ + --clinical-dir Papila/ClinicalData \\ + --manifest manifest.csv \\ + --channels 1 --no-pretrained +""" +from __future__ import annotations + +import argparse +import sys +import time +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import torch +import torch.nn as nn +from sklearn.metrics import accuracy_score, roc_auc_score, roc_curve +from sklearn.model_selection import StratifiedGroupKFold +from torch.utils.data import DataLoader +from tqdm import tqdm + +sys.path.insert(0, str(Path(__file__).resolve().parents[4])) + +from v3.classes.seg_cnn import ( + SegCNN, SegMapDataset, SegMapRecord, + UNetFineTuneDataset, precompute_unet_seg_maps, +) + + +# --------------------------------------------------------------------------- +# Clinical data loader (PAPILA) +# --------------------------------------------------------------------------- + +def load_papila_labels( + clinical_dir: Path, + label_col: str = "Diagnosis", + drop_suspects: bool = True, +) -> pd.DataFrame: + """ + Return a DataFrame with columns: + patient_id (int), eye (str), label (int) + + Reads patient_data_od.xlsx + patient_data_os.xlsx from clinical_dir. + """ + od_path = clinical_dir / "patient_data_od.xlsx" + os_path = clinical_dir / "patient_data_os.xlsx" + frames = [] + for path, eye in ((od_path, "OD"), (os_path, "OS")): + if not path.exists(): + raise FileNotFoundError(f"Clinical data not found: {path}") + df = pd.read_excel(path, header=1) + df["eye"] = eye + id_col = "Patient ID" if "Patient ID" in df.columns else "ID" + df["patient_id"] = ( + df[id_col].astype(str).str.extract(r"(\d+)")[0].astype(int) + ) + frames.append(df) + df = pd.concat(frames, ignore_index=True) + + if drop_suspects: + df = df[df[label_col] != 2].reset_index(drop=True) + + df["label"] = df[label_col].astype(int) + return df[["patient_id", "eye", "label"]].copy() + + +# --------------------------------------------------------------------------- +# Build SegMapRecords from manifest + clinical labels +# --------------------------------------------------------------------------- + +def build_records( + manifest_path: Path, + clinical_df: pd.DataFrame, + dataset_filter: str = "papila", +) -> List[SegMapRecord]: + """ + Join manifest entries (image paths + annotation paths) with clinical labels. + + Returns one SegMapRecord per matched eye sample. + """ + manifest = pd.read_csv(manifest_path) + papila_rows = manifest[manifest["dataset"] == dataset_filter].copy() + + # Parse patient_id and eye from sample_id e.g. "papila_RET042OD" → 42, "OD" + def _parse(sid: str): + sid = sid.replace(f"{dataset_filter}_RET", "") + eye = sid[-2:].upper() # "OD" or "OS" + pid = int(sid[:-2]) + return pid, eye + + papila_rows[["patient_id", "eye"]] = pd.DataFrame( + papila_rows["sample_id"].apply(_parse).tolist(), + index=papila_rows.index, + ) + + merged = papila_rows.merge( + clinical_df[["patient_id", "eye", "label"]], + on=["patient_id", "eye"], + how="inner", + ) + + records: List[SegMapRecord] = [] + for _, row in merged.iterrows(): + records.append( + SegMapRecord( + sample_id=row["sample_id"], + image_path=Path(row["image_path"]), + annotation_disc=Path(row["annotation_disc"]), + annotation_cup=Path(row["annotation_cup"]), + annotation_type_disc=row["annotation_type_disc"], + annotation_type_cup=row["annotation_type_cup"], + patient_id=int(row["patient_id"]), + eye=str(row["eye"]), + label=int(row["label"]), + ) + ) + return records + + +# --------------------------------------------------------------------------- +# Training / evaluation helpers +# --------------------------------------------------------------------------- + +def train_epoch( + model: nn.Module, + loader: DataLoader, + optimizer: torch.optim.Optimizer, + criterion: nn.Module, + device: torch.device, +) -> float: + model.train() + total_loss = 0.0 + n = 0 + for x, y in loader: + x, y = x.to(device), y.to(device) + optimizer.zero_grad() + logits = model(x) + loss = criterion(logits, y) + loss.backward() + optimizer.step() + total_loss += loss.item() * x.size(0) + n += x.size(0) + return total_loss / n if n else float("nan") + + +@torch.no_grad() +def evaluate( + model: nn.Module, + loader: DataLoader, + device: torch.device, +) -> Tuple[float, float, np.ndarray, np.ndarray]: + """Returns (auc, acc, y_true, y_prob).""" + model.eval() + probs_list, labels_list = [], [] + for x, y in loader: + x = x.to(device) + logits = model(x) + prob = torch.softmax(logits, dim=1)[:, 1].cpu().numpy() + probs_list.append(prob) + labels_list.append(y.numpy()) + y_true = np.concatenate(labels_list) + y_prob = np.concatenate(probs_list) + auc = float(roc_auc_score(y_true, y_prob)) if len(np.unique(y_true)) > 1 else float("nan") + acc = float(accuracy_score(y_true, (y_prob >= 0.5).astype(int))) + return auc, acc, y_true, y_prob + + +# --------------------------------------------------------------------------- +# U-Net fine-tuning +# --------------------------------------------------------------------------- + +def finetune_unet( + segmenter, + records: List[SegMapRecord], + epochs: int, + lr: float, + batch_size: int, + device: torch.device, +) -> None: + """Fine-tune the U-Net on a fold's training records using GT annotations.""" + import copy + ds = UNetFineTuneDataset( + records, + target_size=segmenter.target_size, + normalize=segmenter.normalize, + ) + loader = DataLoader(ds, batch_size=batch_size, shuffle=True, num_workers=0) + optimizer = torch.optim.Adam(segmenter.model.parameters(), lr=lr) + criterion = nn.BCEWithLogitsLoss() + + segmenter.model.train() + for epoch in tqdm(range(1, epochs + 1), desc=" U-Net finetune", unit="ep", leave=False): + for images, masks in loader: + images, masks = images.to(device), masks.to(device) + optimizer.zero_grad() + loss = criterion(segmenter.model(images), masks) + loss.backward() + optimizer.step() + segmenter.model.eval() + + +# --------------------------------------------------------------------------- +# Cross-validation loop +# --------------------------------------------------------------------------- + +def _train_one_split( + train_recs, val_recs, y_train, args, device, out_dir, label, + train_seg_maps=None, val_seg_maps=None, +): + """Train one fold/split, return (auc, acc, y_true, y_prob, fpr, tpr).""" + train_ds = SegMapDataset( + train_recs, + target_size=args.img_size, + channels=args.channels, + augment=True, + seg_target_size=args.seg_size, + crop_to_disc=not args.no_crop, + precomputed_seg_maps=train_seg_maps, + ) + val_ds = SegMapDataset( + val_recs, + target_size=args.img_size, + channels=args.channels, + augment=False, + seg_target_size=args.seg_size, + crop_to_disc=not args.no_crop, + precomputed_seg_maps=val_seg_maps, + ) + train_loader = DataLoader( + train_ds, batch_size=args.batch_size, shuffle=True, + num_workers=args.workers, pin_memory=True, + ) + val_loader = DataLoader( + val_ds, batch_size=args.batch_size, shuffle=False, + num_workers=args.workers, pin_memory=True, + ) + + model = SegCNN( + num_classes=2, + backbone=args.backbone, + pretrained=not args.no_pretrained, + in_channels=args.channels, + dropout=args.dropout, + ).to(device) + + n_pos = int(y_train.sum()) + n_neg = len(y_train) - n_pos + class_weights = ( + torch.tensor([1.0, n_neg / n_pos], dtype=torch.float32).to(device) + if n_pos > 0 and n_neg > 0 else None + ) + criterion = nn.CrossEntropyLoss(weight=class_weights) + optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.wd) + scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( + optimizer, T_max=args.epochs, eta_min=args.lr * 0.01 + ) + + best_auc, best_state = -1.0, None + epoch_bar = tqdm(range(1, args.epochs + 1), desc=label, unit="ep") + for _ in epoch_bar: + train_loss = train_epoch(model, train_loader, optimizer, criterion, device) + val_auc, val_acc, _, _ = evaluate(model, val_loader, device) + scheduler.step() + epoch_bar.set_postfix(loss=f"{train_loss:.4f}", auc=f"{val_auc:.3f}", acc=f"{val_acc:.3f}") + if val_auc > best_auc: + best_auc = val_auc + best_state = {k: v.clone() for k, v in model.state_dict().items()} + torch.save({"model": best_state, "auc": best_auc}, out_dir / "best.pt") + + if best_state is not None: + model.load_state_dict(best_state) + final_auc, final_acc, y_true, y_prob = evaluate(model, val_loader, device) + fpr, tpr, _ = roc_curve(y_true, y_prob, pos_label=1) + return final_auc, final_acc, y_true, y_prob, fpr, tpr + + +def run_cv( + records: List[SegMapRecord], + args, + device: torch.device, + unet_segmenter=None, + out_dir: Path = Path("analysis_data/seg_cnn"), +) -> pd.DataFrame: + out_dir.mkdir(parents=True, exist_ok=True) + + y = np.array([r.label for r in records]) + groups = np.array([r.patient_id for r in records]) + splits = list(StratifiedGroupKFold(n_splits=args.n_splits).split( + np.arange(len(records)), y, groups + )) + + # If using U-Net without fine-tuning, precompute all seg maps once upfront. + # If fine-tuning, we must precompute per fold (after fine-tuning) so we + # save the base weights here to restore at the start of each fold. + base_unet_state = None + all_seg_maps = None + if unet_segmenter is not None: + if args.finetune_epochs > 0: + import copy + base_unet_state = copy.deepcopy(unet_segmenter.model.state_dict()) + else: + print(f"Precomputing U-Net seg maps for {len(records)} samples (once for all folds)...") + all_seg_maps = precompute_unet_seg_maps(records, unet_segmenter, args.unet_threshold) + + fold_metrics: List[Dict] = [] + roc_curves = [] + + for fold_idx, (train_idx, val_idx) in enumerate(splits): + start = time.time() + print(f"\n{'='*60}") + print(f" Fold {fold_idx+1}/{args.n_splits} " + f"(train={len(train_idx)}, val={len(val_idx)})") + print(f"{'='*60}") + + fold_dir = out_dir / f"fold{fold_idx}" + fold_dir.mkdir(parents=True, exist_ok=True) + + train_recs = [records[i] for i in train_idx] + val_recs = [records[i] for i in val_idx] + + if unet_segmenter is not None and args.finetune_epochs > 0: + # Restore base REFUGE weights, then fine-tune on this fold's training data only + unet_segmenter.model.load_state_dict(copy.deepcopy(base_unet_state)) + print(f" Fine-tuning U-Net for {args.finetune_epochs} epochs on training fold...") + finetune_unet( + unet_segmenter, train_recs, + epochs=args.finetune_epochs, + lr=args.finetune_lr, + batch_size=args.finetune_batch_size, + device=device, + ) + print(f" Generating seg maps with fine-tuned U-Net...") + train_maps = precompute_unet_seg_maps(train_recs, unet_segmenter, args.unet_threshold) + val_maps = precompute_unet_seg_maps(val_recs, unet_segmenter, args.unet_threshold) + else: + train_maps = [all_seg_maps[i] for i in train_idx] if all_seg_maps else None + val_maps = [all_seg_maps[i] for i in val_idx] if all_seg_maps else None + + auc, acc, y_true, y_prob, fpr, tpr = _train_one_split( + train_recs, val_recs, y[train_idx], args, device, + fold_dir, label=f"Fold {fold_idx+1}", + train_seg_maps=train_maps, val_seg_maps=val_maps, + ) + elapsed = time.time() - start + roc_curves.append((fpr, tpr, auc)) + print(f" Fold {fold_idx+1} AUC={auc:.4f} ACC={acc:.4f} ({elapsed:.0f}s)") + + pd.DataFrame({"y_true": y_true, "y_prob": y_prob}).to_csv( + fold_dir / "val_probs.csv", index=False + ) + fold_metrics.append({ + "fold": fold_idx, "auc": auc, "acc": acc, + "n_train": len(train_idx), "n_val": len(val_idx), "elapsed_s": elapsed, + }) + + metrics_df = pd.DataFrame(fold_metrics) + metrics_df.to_csv(out_dir / "fold_metrics.csv", index=False) + + mean_auc = float(metrics_df["auc"].mean()) + std_auc = float(metrics_df["auc"].std()) + mean_acc = float(metrics_df["acc"].mean()) + std_acc = float(metrics_df["acc"].std()) + + print(f"\n{'='*60}") + print(f" CV Summary ({args.n_splits} folds)") + print(f" AUC = {mean_auc:.4f} ± {std_auc:.4f}") + print(f" ACC = {mean_acc:.4f} ± {std_acc:.4f}") + print(f"{'='*60}\n") + + pd.DataFrame([{ + "backbone": args.backbone, "seg_mode": args.seg_mode, + "channels": args.channels, "pretrained": not args.no_pretrained, + "epochs": args.epochs, "lr": args.lr, "dropout": args.dropout, + "auc_mean": mean_auc, "auc_std": std_auc, + "acc_mean": mean_acc, "acc_std": std_acc, + "n_folds": args.n_splits, + }]).to_csv(out_dir / "summary.csv", index=False) + + # Mean ROC curve + mean_fpr = np.linspace(0, 1, 200) + tprs = [np.interp(mean_fpr, fpr, tpr) for fpr, tpr, _ in roc_curves] + mean_tpr = np.mean(tprs, axis=0); mean_tpr[-1] = 1.0 + std_tpr = np.std(tprs, axis=0) + fig, ax = plt.subplots(figsize=(5.5, 4.5)) + ax.plot(mean_fpr, mean_tpr, lw=2, + label=f"AUC = {mean_auc:.3f} ± {std_auc:.3f}") + ax.fill_between(mean_fpr, + np.maximum(mean_tpr - std_tpr, 0), + np.minimum(mean_tpr + std_tpr, 1), + alpha=0.2, color="steelblue") + ax.plot([0, 1], [0, 1], "k--", lw=1) + ax.set_xlabel("False Positive Rate"); ax.set_ylabel("True Positive Rate") + ax.set_title(f"Seg-map CNN ({args.backbone}, {args.seg_mode})") + ax.legend(loc="lower right"); ax.grid(True, alpha=0.3, linestyle="--") + fig.tight_layout(); fig.savefig(out_dir / "roc_mean.png", dpi=170); plt.close(fig) + + return metrics_df + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + description="Train a CNN on optic disc/cup segmentation maps.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + # --- Data paths --- + p.add_argument("--image-dir", required=True, help="PAPILA FundusImages directory") + p.add_argument("--clinical-dir", required=True, help="PAPILA ClinicalData directory") + p.add_argument("--manifest", required=True, help="manifest.csv with annotation paths") + p.add_argument("--output-dir", default="analysis_data/seg_cnn", + help="Where to save results") + + # --- Segmentation mode --- + p.add_argument("--seg-mode", choices=["gt", "unet"], default="gt", + help="gt = expert annotations; unet = predicted masks from trained U-Net") + p.add_argument("--unet-weights", default=None, + help="Path to trained UNet weights (.pt); required for --seg-mode unet") + p.add_argument("--unet-normalize", default="per_image", + choices=["none", "per_image", "imagenet"], + help="Normalisation used when the UNet was trained") + p.add_argument("--unet-threshold", type=float, default=0.5, + help="Sigmoid threshold for binary mask from U-Net logits") + p.add_argument("--seg-size", type=int, default=512, + help="Spatial size at which GT contours are rasterised / U-Net runs") + + # --- Model --- + p.add_argument("--backbone", default="resnet18", + choices=["resnet18", "resnet50", "efficientnet_b0"], + help="CNN backbone") + p.add_argument("--channels", type=int, default=3, choices=[1, 3], + help="1 = single-channel normalised label map; " + "3 = one-hot [bg, disc_rim, cup]") + p.add_argument("--no-pretrained", action="store_true", + help="Do not load ImageNet weights for the backbone") + p.add_argument("--dropout", type=float, default=0.3) + + # --- Training --- + p.add_argument("--epochs", type=int, default=60) + p.add_argument("--batch-size", type=int, default=16) + p.add_argument("--lr", type=float, default=1e-4) + p.add_argument("--wd", type=float, default=1e-4, + help="AdamW weight decay") + p.add_argument("--img-size", type=int, default=224, + help="CNN input spatial resolution") + p.add_argument("--no-crop", action="store_true", + help="Disable disc-region cropping (keeps full-image seg map)") + p.add_argument("--workers", type=int, default=4, + help="DataLoader num_workers") + + # --- U-Net fine-tuning (only applies with --seg-mode unet) --- + p.add_argument("--finetune-epochs", type=int, default=0, + help="Epochs to fine-tune U-Net on each fold's training data " + "(0 = disabled, uses base REFUGE weights as-is)") + p.add_argument("--finetune-lr", type=float, default=1e-5, + help="Learning rate for U-Net fine-tuning") + p.add_argument("--finetune-batch-size", type=int, default=4, + help="Batch size for U-Net fine-tuning") + + # --- CV --- + p.add_argument("--n-splits", type=int, default=5, help="Number of CV folds") + p.add_argument("--seed", type=int, default=42) + + # --- Misc --- + p.add_argument("--device", default=None, + help="torch device string (default: cuda if available)") + p.add_argument("--label-col", default="Diagnosis", + help="Label column in PAPILA clinical xlsx") + + return p + + +def main(argv=None): + args = build_parser().parse_args(argv) + + torch.manual_seed(args.seed) + np.random.seed(args.seed) + + device = torch.device( + args.device if args.device + else ("cuda" if torch.cuda.is_available() else "cpu") + ) + print(f"Device: {device}") + + # ---- Load PAPILA labels ---- + clinical_df = load_papila_labels( + Path(args.clinical_dir), + label_col=args.label_col, + drop_suspects=True, + ) + print(f"Clinical labels loaded: {len(clinical_df)} eye records " + f"(N={int((clinical_df.label==0).sum())}, G={int((clinical_df.label==1).sum())})") + + # ---- Build records ---- + records = build_records(Path(args.manifest), clinical_df) + print(f"Matched records: {len(records)} " + f"(N={sum(r.label==0 for r in records)}, G={sum(r.label==1 for r in records)})") + + if not records: + print("ERROR: No records matched. Check manifest and clinical data paths.") + sys.exit(1) + + # ---- Load UNet if needed ---- + unet_segmenter = None + if args.seg_mode == "unet": + if not args.unet_weights: + print("ERROR: --seg-mode unet requires --unet-weights") + sys.exit(1) + from v3.classes.unet_segmenter import UNetSegmenter # noqa + import tempfile, csv, os + + # Build a minimal manifest for UNetSegmenter init + fd, tmp = tempfile.mkstemp(suffix=".csv") + with os.fdopen(fd, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow([ + "sample_id", "dataset", "image_path", + "annotation_disc", "annotation_cup", + "annotation_type_disc", "annotation_type_cup", "split", + ]) + r = records[0] + writer.writerow([ + r.sample_id, "papila", str(r.image_path), + str(r.annotation_disc), str(r.annotation_cup), + r.annotation_type_disc, r.annotation_type_cup, "train", + ]) + unet_segmenter = UNetSegmenter( + manifest_path=Path(tmp), + normalize=args.unet_normalize, + target_size=args.seg_size, + ) + os.unlink(tmp) + state = torch.load(args.unet_weights, map_location=unet_segmenter.device) + state_dict = state.get("model", state) + unet_segmenter.model.load_state_dict(state_dict) + unet_segmenter.model.to(unet_segmenter.device) + unet_segmenter.model.eval() + print(f"U-Net weights loaded from {args.unet_weights}") + + # ---- Run CV ---- + out_dir = Path(args.output_dir) + run_cv( + records=records, + args=args, + device=device, + unet_segmenter=unet_segmenter, + out_dir=out_dir, + ) + + +if __name__ == "__main__": + main() diff --git a/v3/scripts/main/phase1/__init__.py b/v3/scripts/main/phase1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/v3/scripts/main/phase_1_papila_reproduce.py b/v3/scripts/main/phase1/phase_1_papila_reproduce.py similarity index 100% rename from v3/scripts/main/phase_1_papila_reproduce.py rename to v3/scripts/main/phase1/phase_1_papila_reproduce.py diff --git a/v3/scripts/main/phase2/__init__.py b/v3/scripts/main/phase2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/v3/scripts/main/phase2/phase_2_overnight.sh b/v3/scripts/main/phase2/phase_2_overnight.sh new file mode 100755 index 0000000..adb1614 --- /dev/null +++ b/v3/scripts/main/phase2/phase_2_overnight.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# Phase 2 overnight batch — image-only ResNet50, 10x5 rep-CV +# Runs 6 configurations: +# 1. Leaky CV (eye-level splits) +# 2. Proper CV (patient-level, baseline) +# 3. GT crop scale=1.1 (paper-matched tight crop) +# 4. GT crop scale=2.5 (default generous crop) +# 5. UNet crop scale=1.1 +# 6. UNet crop scale=2.5 +set -euo pipefail + +SCRIPT="python -m v3.scripts.main.run_cv" +OUTROOT="v3/results/phase2" +MANIFEST="manifest.csv" +UNET_WEIGHTS="models/v2/refuge/segmentation/per_image/best.pt" + +BASE="--eval-mode binary \ + --tower-mode single \ + --bridge-mode image_only \ + --backbone resnet50 \ + --epochs 30 \ + --augment \ + --in-memory-cache \ + --reps 10 \ + --rep-seed-start 100 \ + --rep-seed-step 100 \ + --output-root ${OUTROOT}" + +echo "============================================================" +echo " Phase 2 overnight batch" +echo " $(date)" +echo "============================================================" + +# ---------------------------------------------------------------- +# 1. Leaky CV (eye-level splits, no crop) +# ---------------------------------------------------------------- +echo "" +echo "=== [1/6] Leaky CV (eye-level) ===" +$SCRIPT $BASE \ + --leaky-cv \ + --run-name imageonly_resnet50_leaky + +# ---------------------------------------------------------------- +# 2. Proper CV (patient-level, no crop) — baseline +# ---------------------------------------------------------------- +echo "" +echo "=== [2/6] Proper CV (patient-level, baseline) ===" +$SCRIPT $BASE \ + --run-name imageonly_resnet50_proper + +# ---------------------------------------------------------------- +# 3. GT crop, scale=1.1 (paper-matched tight crop) +# ---------------------------------------------------------------- +echo "" +echo "=== [3/6] GT crop, scale=1.1 ===" +$SCRIPT $BASE \ + --img-crop-gt \ + --img-crop-manifest ${MANIFEST} \ + --img-crop-scale 1.1 \ + --img-crop-size 200 \ + --run-name imageonly_resnet50_gtcrop_1.1 + +# ---------------------------------------------------------------- +# 4. GT crop, scale=2.5 (default generous crop) +# ---------------------------------------------------------------- +echo "" +echo "=== [4/6] GT crop, scale=2.5 ===" +$SCRIPT $BASE \ + --img-crop-gt \ + --img-crop-manifest ${MANIFEST} \ + --img-crop-scale 2.5 \ + --img-crop-size 200 \ + --run-name imageonly_resnet50_gtcrop_2.5 + +# ---------------------------------------------------------------- +# 5. UNet crop, scale=1.1 +# ---------------------------------------------------------------- +echo "" +echo "=== [5/6] UNet crop, scale=1.1 ===" +$SCRIPT $BASE \ + --img-crop-weights ${UNET_WEIGHTS} \ + --img-crop-manifest ${MANIFEST} \ + --img-crop-scale 1.1 \ + --img-crop-size 200 \ + --run-name imageonly_resnet50_unetcrop_1.1 + +# ---------------------------------------------------------------- +# 6. UNet crop, scale=2.5 +# ---------------------------------------------------------------- +echo "" +echo "=== [6/6] UNet crop, scale=2.5 ===" +$SCRIPT $BASE \ + --img-crop-weights ${UNET_WEIGHTS} \ + --img-crop-manifest ${MANIFEST} \ + --img-crop-scale 2.5 \ + --img-crop-size 200 \ + --run-name imageonly_resnet50_unetcrop_2.5 + +# ---------------------------------------------------------------- +# 7. Refugelike backbone (proper CV, no crop) — pre-training effect +# ---------------------------------------------------------------- +echo "" +echo "=== [7/7] Refugelike backbone (proper CV, no crop) ===" +$SCRIPT $BASE \ + --backbone refugelike \ + --run-name imageonly_refugelike_proper + +echo "" +echo "============================================================" +echo " All done — $(date)" +echo "============================================================" diff --git a/v3/scripts/main/phase3/__init__.py b/v3/scripts/main/phase3/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/v3/scripts/main/phase3/dispatch_phase3.py b/v3/scripts/main/phase3/dispatch_phase3.py new file mode 100644 index 0000000..9d38436 --- /dev/null +++ b/v3/scripts/main/phase3/dispatch_phase3.py @@ -0,0 +1,197 @@ +""" +Dispatch phase 3 experiment runs to the distributed job server. + +Reads experiment_grid.json, checks which runs already have complete 10x5 results, +and submits the rest via submit-cv. Skips runs marked needs_implementation. + +Usage: + python -m v3.scripts.main.phase3.dispatch_phase3 \ + --server http://hades:8765 --token hypertower + + # Dry run (print what would be submitted, don't actually submit): + python -m v3.scripts.main.phase3.dispatch_phase3 \ + --server http://hades:8765 --token hypertower --dry-run + + # Override number of reps (default 10): + python -m v3.scripts.main.phase3.dispatch_phase3 \ + --server http://hades:8765 --token hypertower --reps 4 +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +import requests + +# Allow running as `python v3/scripts/main/phase3/dispatch_phase3.py` +sys.path.insert(0, str(Path(__file__).resolve().parents[4])) + +GRID_PATH = Path(__file__).parent / "experiment_grid.json" +RESULTS_ROOT = Path(__file__).resolve().parents[4] / "v3" / "results" +MODULE = "v3.scripts.main.run_cv" +OUTPUT_DIR = "v3/results" +REP_SEED_START = 100 +REP_SEED_STEP = 100 + + +# ── Completion check ────────────────────────────────────────────────────────── + +def _completed_reps(run_name: str, reps: int) -> list[int]: + """Return list of rep indices that already have a summary.json.""" + done = [] + for i in range(reps): + summary = RESULTS_ROOT / run_name / f"rep{i:02d}" / "binary" / "single" / "summary.json" + if summary.exists(): + done.append(i) + return done + + +# ── Server API ──────────────────────────────────────────────────────────────── + +class _API: + def __init__(self, base_url: str, token: str): + self.base_url = base_url.rstrip("/") + self._h = {"x-token": token} + + def get(self, path: str, **params) -> object: + r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10) + r.raise_for_status() + return r.json() + + def post(self, path: str, body: dict) -> dict: + r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10) + r.raise_for_status() + return r.json() + + +def _queued_reps(jobs: list[dict], run_name: str) -> set[int]: + """Return rep indices already pending or running in the server queue.""" + active = set() + for job in jobs: + if job["run_name"] != run_name: + continue + if job["state"] not in ("pending", "running"): + continue + # Extract --rep-index from job args + try: + args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"]) + if "--rep-index" in args: + active.add(int(args[args.index("--rep-index") + 1])) + except Exception: + pass + return active + + +def _submit_cv(api: _API, run_name: str, run_args: list[str], + reps: int, missing: list[int], dry_run: bool): + """Submit one job per missing rep.""" + for i in missing: + seed = REP_SEED_START + i * REP_SEED_STEP + rep_args = run_args + [ + "--run-name", run_name, + "--reps", "1", + "--rep-seed-start", str(seed), + "--rep-index", str(i), + ] + body = { + "run_name": run_name, + "module": MODULE, + "args": rep_args, + "output_dir": OUTPUT_DIR, + "priority": 0, + } + if dry_run: + print(f" [dry-run] would queue rep{i:02d} seed={seed}") + else: + resp = api.post("/jobs", body) + print(f" queued rep{i:02d} seed={seed} job_id={resp['job_id']}") + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""), + help="Server URL (or set HT_SERVER)") + ap.add_argument("--token", default=os.environ.get("HT_TOKEN", ""), + help="Shared secret (or set HT_TOKEN)") + ap.add_argument("--reps", type=int, default=10, + help="Expected number of reps per run (default: 10)") + ap.add_argument("--grid", type=Path, default=GRID_PATH, + help="Path to experiment grid JSON (default: experiment_grid.json)") + ap.add_argument("--dry-run", action="store_true", + help="Print what would be submitted without actually submitting") + args = ap.parse_args() + + if not args.dry_run: + if not args.server: + ap.error("--server is required (or set HT_SERVER)") + if not args.token: + ap.error("--token is required (or set HT_TOKEN)") + elif not args.server or not args.token: + print("[dry-run] no --server/--token provided — skipping queue check, showing disk state only") + + grid = json.loads(args.grid.read_text()) + common_args = grid["common_args"] + api = _API(args.server, args.token) if (args.server and args.token) else None + + # Fetch current server queue once (pending + running) + server_jobs: list[dict] = [] + if api: + try: + all_jobs = api.get("/jobs") + server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")] + print(f"[server] {len(server_jobs)} job(s) currently pending/running in queue") + except Exception as e: + print(f"[warn] could not fetch server queue: {e}") + + # Collect all runs: baseline + every group's runs + all_runs = [grid["baseline"]] + for group in grid["groups"]: + if group.get("needs_implementation"): + print(f"\n[skip] group '{group['name']}' — {group['needs_implementation']}") + continue + all_runs.extend(group["runs"]) + + submitted_total = 0 + skipped_total = 0 + + for run in all_runs: + run_name = run["run_name"] + run_args = common_args + run.get("extra_args", []) + done = set(_completed_reps(run_name, args.reps)) + queued = _queued_reps(server_jobs, run_name) + accounted = done | queued + missing = [i for i in range(args.reps) if i not in accounted] + + if not missing: + if len(done) == args.reps: + print(f"\n[done] {run_name} ({args.reps}/{args.reps} reps complete)") + else: + in_q = sorted(queued - done) + print(f"\n[skip] {run_name} ({len(done)} done, {len(in_q)} queued: {[f'rep{i:02d}' for i in in_q]})") + skipped_total += 1 + continue + + parts = [] + if done: parts.append(f"{len(done)} done") + if queued: parts.append(f"{len(queued - done)} queued") + status = ", ".join(parts) if parts else "not started" + print(f"\n[queue] {run_name} ({status}) — submitting {len(missing)} rep(s)") + _submit_cv(api, run_name, run_args, args.reps, missing, args.dry_run) + submitted_total += len(missing) + + print(f"\n{'='*50}") + print(f"Submitted: {submitted_total} jobs | Already accounted for: {skipped_total} runs") + if grid.get("groups"): + needs_impl = sum(1 for g in grid["groups"] if g.get("needs_implementation")) + if needs_impl: + print(f"Skipped (needs implementation): {needs_impl} group(s)") + + +if __name__ == "__main__": + main() diff --git a/v3/scripts/main/phase3/epoch_grid.json b/v3/scripts/main/phase3/epoch_grid.json new file mode 100644 index 0000000..b2d8d5f --- /dev/null +++ b/v3/scripts/main/phase3/epoch_grid.json @@ -0,0 +1,66 @@ +{ + "_notes": [ + "Epoch length sensitivity experiments.", + "All other settings match the phase3 baseline (fused bridge, BCD p=0.5, refugelike, etc.).", + "common_args are prepended to every run's args list." + ], + + "common_args": [ + "--eval-mode", "binary", + "--tower-mode", "single", + "--in-memory-cache", + "--augment", + "--tune-binary-threshold", + "--backbone", "refugelike", + "--output-root", "v3/results" + ], + + "_common_args_implicit_defaults": { + "--bridge-mode": "fused", + "--tower-loss-mode": "bcd", + "--bcd-prob": "0.5", + "--warmup-cd-epochs": "40", + "--single-warmup-tower-epochs": "3", + "--single-warmup-fused-epochs": "3" + }, + + "baseline": { + "run_name": "phase3/epochs_30", + "description": "30-epoch run — same as phase3 baseline, included here for direct comparison.", + "extra_args": ["--epochs", "30"] + }, + + "groups": [ + { + "name": "epoch_length", + "description": "Test sensitivity to total training epochs (warmup epochs unchanged).", + "runs": [ + { + "run_name": "phase3/epochs_1", + "description": "1 epoch total — effectively pure warmup output with a single main-phase step.", + "extra_args": ["--epochs", "1"] + }, + { + "run_name": "phase3/epochs_5", + "description": "5 epochs total.", + "extra_args": ["--epochs", "5"] + }, + { + "run_name": "phase3/epochs_10", + "description": "10 epochs total.", + "extra_args": ["--epochs", "10"] + }, + { + "run_name": "phase3/epochs_20", + "description": "20 epochs total.", + "extra_args": ["--epochs", "20"] + }, + { + "run_name": "phase3/epochs_50", + "description": "50 epochs total.", + "extra_args": ["--epochs", "50"] + } + ] + } + ] +} diff --git a/v3/scripts/main/phase3/experiment_grid.json b/v3/scripts/main/phase3/experiment_grid.json new file mode 100644 index 0000000..f7a87a1 --- /dev/null +++ b/v3/scripts/main/phase3/experiment_grid.json @@ -0,0 +1,362 @@ +{ + "_notes": [ + "All runs use single-eye tower mode, binary eval, 10x5 rep-CV.", + "common_args are prepended to every run's args list.", + "Entries marked 'needs_implementation' require small code changes before running (noted inline).", + "Bridge fusion uses elementwise product of projected image/clinical features.", + "SE infrastructure exists in Bridge/ImageTower/ClinicalTower but use_se is hardcoded False", + " in SingleEyeHT \u2014 add --se-img-tower / --se-cd-tower / --se-bridge flags to wire through.", + "Dropout is hardcoded: Bridge classifier=0.5, ClinicalTower=0.1 \u2014 add --bridge-dropout /", + " --cd-dropout flags to make configurable." + ], + "common_args": [ + "--eval-mode", + "binary", + "--tower-mode", + "single", + "--epochs", + "30", + "--in-memory-cache", + "--augment", + "--tune-binary-threshold", + "--backbone", + "refugelike", + "--output-root", + "v3/results" + ], + "_common_args_implicit_defaults": { + "--bridge-mode": "fused", + "--tower-loss-mode": "bcd", + "--bcd-prob": "0.5", + "--warmup-cd-epochs": "40", + "--single-warmup-tower-epochs": "3", + "--single-warmup-fused-epochs": "3" + }, + "baseline": { + "run_name": "phase3/baseline", + "description": "Fused bridge (image+clinical), BCD p=0.5, cd_warmup=40, tower/fused warmup=3/3, no SE, no IOP correction.", + "extra_args": [] + }, + "groups": [ + { + "name": "loss_function", + "description": "Test BCD loss variants vs cross-entropy baseline.", + "runs": [ + { + "run_name": "phase3/loss_all", + "description": "All-losses mode (cross-entropy on all three heads every step).", + "extra_args": [ + "--tower-loss-mode", + "all" + ] + }, + { + "run_name": "phase3/loss_bcd_p03", + "description": "BCD with lower switching probability (more CE, less BCD).", + "extra_args": [ + "--tower-loss-mode", + "bcd", + "--bcd-prob", + "0.3" + ] + }, + { + "run_name": "phase3/loss_bcd_p07", + "description": "BCD with higher switching probability (more BCD, less CE).", + "extra_args": [ + "--tower-loss-mode", + "bcd", + "--bcd-prob", + "0.7" + ] + } + ] + }, + { + "name": "se_attention", + "description": "Squeeze-and-excitation gates at different points in the network.", + "runs": [ + { + "run_name": "phase3/se_bridge", + "description": "SE gate on fused vector inside the bridge only.", + "extra_args": [ + "--se-bridge" + ] + }, + { + "run_name": "phase3/se_img_tower", + "description": "SE gate on image tower output features.", + "extra_args": [ + "--se-img-tower" + ] + }, + { + "run_name": "phase3/se_cd_tower", + "description": "SE gate on clinical tower output features.", + "extra_args": [ + "--se-cd-tower" + ] + }, + { + "run_name": "phase3/se_all", + "description": "SE gates on image tower, clinical tower, and bridge.", + "extra_args": [ + "--se-img-tower", + "--se-cd-tower", + "--se-bridge" + ] + } + ] + }, + { + "name": "iop_correction", + "description": "Test different IOP measurement correction strategies (default: no correction).", + "runs": [ + { + "run_name": "phase3/iop_ratio", + "description": "IOP correction via Perkins\u2192Pneumatic ratio scaling.", + "extra_args": [ + "--iop-corr-method", + "ratio" + ] + }, + { + "run_name": "phase3/iop_ols", + "description": "IOP correction via OLS regression.", + "extra_args": [ + "--iop-corr-method", + "ols" + ] + }, + { + "run_name": "phase3/iop_lad", + "description": "IOP correction via LAD (robust to outliers) regression.", + "extra_args": [ + "--iop-corr-method", + "lad" + ] + }, + { + "run_name": "phase3/iop_multi", + "description": "IOP correction via multivariate regression including pachymetry.", + "extra_args": [ + "--iop-corr-method", + "multi" + ] + }, + { + "run_name": "phase3/iop_ratio_drop_raw", + "description": "Ratio correction + drop raw IOP (only corrected IOP seen by model).", + "extra_args": [ + "--iop-corr-method", + "ratio", + "--iop-drop-raw" + ] + } + ] + }, + { + "name": "feature_ablation", + "description": "Exclude individual clinical features to measure each one's contribution.", + "runs": [ + { + "run_name": "phase3/excl_iop", + "description": "No IOP features \u2014 tests how much intraocular pressure contributes.", + "extra_args": [ + "--exclude-cols", + "IOP", + "Pachymetry" + ] + }, + { + "run_name": "phase3/excl_age", + "description": "No age feature.", + "extra_args": [ + "--exclude-cols", + "Age" + ] + }, + { + "run_name": "phase3/excl_axial_length", + "description": "No axial length feature.", + "extra_args": [ + "--exclude-cols", + "Axial_Length" + ] + }, + { + "run_name": "phase3/excl_refractive", + "description": "No refractive defect feature.", + "extra_args": [ + "--exclude-cols", + "Refractive_Defect" + ] + } + ] + }, + { + "name": "network_dims", + "description": "Test sensitivity to clinical tower and bridge fusion dimensionality.", + "runs": [ + { + "run_name": "phase3/cd_hidden_64", + "description": "Smaller clinical tower (64 hidden units vs default 128).", + "extra_args": [ + "--cd-hidden-dim", + "64" + ] + }, + { + "run_name": "phase3/cd_hidden_256", + "description": "Larger clinical tower (256 hidden units vs default 128).", + "extra_args": [ + "--cd-hidden-dim", + "256" + ] + }, + { + "run_name": "phase3/fusion_dim_128", + "description": "Smaller fusion space (128 vs default 256).", + "extra_args": [ + "--fusion-dim", + "128" + ] + }, + { + "run_name": "phase3/fusion_dim_512", + "description": "Larger fusion space (512 vs default 256).", + "extra_args": [ + "--fusion-dim", + "512" + ] + } + ] + }, + { + "name": "backbone_freezing", + "description": "Partial backbone freezing to reduce overfitting and speed training.", + "runs": [ + { + "run_name": "phase3/freeze_25", + "description": "Freeze earliest 25% of backbone blocks.", + "extra_args": [ + "--freeze-ratio", + "0.25" + ] + }, + { + "run_name": "phase3/freeze_50", + "description": "Freeze earliest 50% of backbone blocks.", + "extra_args": [ + "--freeze-ratio", + "0.50" + ] + } + ] + }, + { + "name": "learning_rate", + "description": "Test LR sensitivity (default 1e-4).", + "runs": [ + { + "run_name": "phase3/lr_1e3", + "description": "Higher learning rate 1e-3.", + "extra_args": [ + "--lr", + "1e-3" + ] + }, + { + "run_name": "phase3/lr_3e4", + "description": "Intermediate learning rate 3e-4.", + "extra_args": [ + "--lr", + "3e-4" + ] + }, + { + "run_name": "phase3/lr_1e5", + "description": "Lower learning rate 1e-5.", + "extra_args": [ + "--lr", + "1e-5" + ] + } + ] + }, + { + "name": "dropout", + "description": "Test bridge classifier and clinical tower dropout rates.", + "runs": [ + { + "run_name": "phase3/bridge_dropout_03", + "description": "Reduce bridge classifier dropout from 0.5 to 0.3.", + "extra_args": [ + "--bridge-dropout", + "0.3" + ] + }, + { + "run_name": "phase3/bridge_dropout_07", + "description": "Increase bridge classifier dropout to 0.7.", + "extra_args": [ + "--bridge-dropout", + "0.7" + ] + }, + { + "run_name": "phase3/cd_dropout_03", + "description": "Increase clinical tower dropout from 0.1 to 0.3.", + "extra_args": [ + "--cd-dropout", + "0.3" + ] + } + ] + }, + { + "name": "warmup", + "description": "Test warmup ablations vs default (cd=40, tower/fused=3/3).", + "runs": [ + { + "run_name": "phase3/warmup_no_cd", + "description": "No CD warmup (cd=0) \u2014 tests whether the 40-epoch CD warmup is necessary.", + "extra_args": [ + "--warmup-cd-epochs", + "0" + ] + }, + { + "run_name": "phase3/warmup_tower5_fused5", + "description": "Extended tower/fused warmup (5/5 vs default 3/3).", + "extra_args": [ + "--single-warmup-tower-epochs", + "5", + "--single-warmup-fused-epochs", + "5" + ] + } + ] + }, + { + "name": "sampling_augmentation", + "description": "Test data sampling and augmentation choices.", + "runs": [ + { + "run_name": "phase3/no_augment", + "description": "No augmentation \u2014 baseline images only.", + "extra_args": [ + "--no-augment" + ] + }, + { + "run_name": "phase3/balanced_sampling", + "description": "Weighted balanced sampler to counter class imbalance.", + "extra_args": [ + "--balanced-sampling" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/v3/scripts/main/phase3/phase35_grid.json b/v3/scripts/main/phase3/phase35_grid.json new file mode 100644 index 0000000..4c0d594 --- /dev/null +++ b/v3/scripts/main/phase3/phase35_grid.json @@ -0,0 +1,54 @@ +{ + "_notes": [ + "Phase 3.5 — confirmation and BCD tuning.", + "All runs use best settings from phase 3: refugelike backbone, ratio IOP correction, drop raw IOP, exclude axial length.", + "Baseline here is phase3/iop_ratio_drop_raw (0.8685 ± 0.011) — already complete, not re-run.", + "common_args are prepended to every run's args list." + ], + + "common_args": [ + "--eval-mode", "binary", + "--tower-mode", "single", + "--epochs", "30", + "--in-memory-cache", + "--augment", + "--tune-binary-threshold", + "--backbone", "refugelike", + "--iop-corr-method", "ratio", + "--iop-drop-raw", + "--exclude-cols", "Axial_Length", + "--output-root", "v3/results" + ], + + "_common_args_implicit_defaults": { + "--bridge-mode": "fused", + "--warmup-cd-epochs": "40", + "--single-warmup-tower-epochs": "3", + "--single-warmup-fused-epochs": "3" + }, + + "baseline": { + "run_name": "phase35/iop_bcd_p07", + "description": "Best IOP preprocessing + best BCD prob from phase 3 combined.", + "extra_args": ["--tower-loss-mode", "bcd", "--bcd-prob", "0.7"] + }, + + "groups": [ + { + "name": "bcd_tuning", + "description": "Extended BCD probability sweep with best IOP settings.", + "runs": [ + { + "run_name": "phase35/iop_bcd_p08", + "description": "BCD p=0.8 with ratio IOP + drop raw.", + "extra_args": ["--tower-loss-mode", "bcd", "--bcd-prob", "0.8"] + }, + { + "run_name": "phase35/iop_bcd_p09", + "description": "BCD p=0.9 with ratio IOP + drop raw.", + "extra_args": ["--tower-loss-mode", "bcd", "--bcd-prob", "0.9"] + } + ] + } + ] +} diff --git a/v3/scripts/main/phase4/__init__.py b/v3/scripts/main/phase4/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/v3/scripts/main/phase4/dispatch_phase4.py b/v3/scripts/main/phase4/dispatch_phase4.py new file mode 100644 index 0000000..2529e01 --- /dev/null +++ b/v3/scripts/main/phase4/dispatch_phase4.py @@ -0,0 +1,199 @@ +""" +Dispatch phase 4 experiment runs to the distributed job server. + +Reads experiment_grid.json, checks which runs already have complete 10x5 results, +and submits the rest. Skips runs marked needs_implementation. + +Usage: + python -m v3.scripts.main.phase4.dispatch_phase4 \ + --server http://hades:8765 --token hypertower + + # Dry run (print what would be submitted, don't actually submit): + python -m v3.scripts.main.phase4.dispatch_phase4 \ + --server http://hades:8765 --token hypertower --dry-run + + # Override number of reps (default 10): + python -m v3.scripts.main.phase4.dispatch_phase4 \ + --server http://hades:8765 --token hypertower --reps 4 +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +import requests + +# Allow running as `python v3/scripts/main/phase3/dispatch_phase3.py` +sys.path.insert(0, str(Path(__file__).resolve().parents[4])) + +GRID_PATH = Path(__file__).parent / "experiment_grid.json" +RESULTS_ROOT = Path(__file__).resolve().parents[4] / "v3" / "results" +MODULE = "v3.scripts.main.run_cv" +OUTPUT_DIR = "v3/results" +REP_SEED_START = 100 +REP_SEED_STEP = 100 + + +# ── Completion check ────────────────────────────────────────────────────────── + +def _completed_reps(run_name: str, reps: int) -> list[int]: + """Return list of rep indices that already have a summary.json (any tower mode).""" + done = [] + for i in range(reps): + rep_dir = RESULTS_ROOT / run_name / f"rep{i:02d}" / "binary" + # Accept any tower mode subdir + if rep_dir.exists() and any((rep_dir / tm / "summary.json").exists() + for tm in ("single", "bilateral", "siamese", "ensemble")): + done.append(i) + return done + + +# ── Server API ──────────────────────────────────────────────────────────────── + +class _API: + def __init__(self, base_url: str, token: str): + self.base_url = base_url.rstrip("/") + self._h = {"x-token": token} + + def get(self, path: str, **params) -> object: + r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10) + r.raise_for_status() + return r.json() + + def post(self, path: str, body: dict) -> dict: + r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10) + r.raise_for_status() + return r.json() + + +def _queued_reps(jobs: list[dict], run_name: str) -> set[int]: + """Return rep indices already pending or running in the server queue.""" + active = set() + for job in jobs: + if job["run_name"] != run_name: + continue + if job["state"] not in ("pending", "running"): + continue + # Extract --rep-index from job args + try: + args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"]) + if "--rep-index" in args: + active.add(int(args[args.index("--rep-index") + 1])) + except Exception: + pass + return active + + +def _submit_cv(api: _API, run_name: str, run_args: list[str], + reps: int, missing: list[int], dry_run: bool): + """Submit one job per missing rep.""" + for i in missing: + seed = REP_SEED_START + i * REP_SEED_STEP + rep_args = run_args + [ + "--run-name", run_name, + "--reps", "1", + "--rep-seed-start", str(seed), + "--rep-index", str(i), + ] + body = { + "run_name": run_name, + "module": MODULE, + "args": rep_args, + "output_dir": OUTPUT_DIR, + "priority": 0, + } + if dry_run: + print(f" [dry-run] would queue rep{i:02d} seed={seed}") + else: + resp = api.post("/jobs", body) + print(f" queued rep{i:02d} seed={seed} job_id={resp['job_id']}") + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""), + help="Server URL (or set HT_SERVER)") + ap.add_argument("--token", default=os.environ.get("HT_TOKEN", ""), + help="Shared secret (or set HT_TOKEN)") + ap.add_argument("--reps", type=int, default=10, + help="Expected number of reps per run (default: 10)") + ap.add_argument("--grid", type=Path, default=GRID_PATH, + help="Path to experiment grid JSON (default: experiment_grid.json)") + ap.add_argument("--dry-run", action="store_true", + help="Print what would be submitted without actually submitting") + args = ap.parse_args() + + if not args.dry_run: + if not args.server: + ap.error("--server is required (or set HT_SERVER)") + if not args.token: + ap.error("--token is required (or set HT_TOKEN)") + elif not args.server or not args.token: + print("[dry-run] no --server/--token provided — skipping queue check, showing disk state only") + + grid = json.loads(args.grid.read_text()) + common_args = grid["common_args"] + api = _API(args.server, args.token) if (args.server and args.token) else None + + # Fetch current server queue once (pending + running) + server_jobs: list[dict] = [] + if api: + try: + all_jobs = api.get("/jobs") + server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")] + print(f"[server] {len(server_jobs)} job(s) currently pending/running in queue") + except Exception as e: + print(f"[warn] could not fetch server queue: {e}") + + # Collect all runs: baseline + every group's runs + all_runs = [grid["baseline"]] + for group in grid["groups"]: + if group.get("needs_implementation"): + print(f"\n[skip] group '{group['name']}' — {group['needs_implementation']}") + continue + all_runs.extend(group["runs"]) + + submitted_total = 0 + skipped_total = 0 + + for run in all_runs: + run_name = run["run_name"] + run_args = common_args + run.get("extra_args", []) + done = set(_completed_reps(run_name, args.reps)) + queued = _queued_reps(server_jobs, run_name) + accounted = done | queued + missing = [i for i in range(args.reps) if i not in accounted] + + if not missing: + if len(done) == args.reps: + print(f"\n[done] {run_name} ({args.reps}/{args.reps} reps complete)") + else: + in_q = sorted(queued - done) + print(f"\n[skip] {run_name} ({len(done)} done, {len(in_q)} queued: {[f'rep{i:02d}' for i in in_q]})") + skipped_total += 1 + continue + + parts = [] + if done: parts.append(f"{len(done)} done") + if queued: parts.append(f"{len(queued - done)} queued") + status = ", ".join(parts) if parts else "not started" + print(f"\n[queue] {run_name} ({status}) — submitting {len(missing)} rep(s)") + _submit_cv(api, run_name, run_args, args.reps, missing, args.dry_run) + submitted_total += len(missing) + + print(f"\n{'='*50}") + print(f"Submitted: {submitted_total} jobs | Already accounted for: {skipped_total} runs") + if grid.get("groups"): + needs_impl = sum(1 for g in grid["groups"] if g.get("needs_implementation")) + if needs_impl: + print(f"Skipped (needs implementation): {needs_impl} group(s)") + + +if __name__ == "__main__": + main() diff --git a/v3/scripts/main/phase4/experiment_grid.json b/v3/scripts/main/phase4/experiment_grid.json new file mode 100644 index 0000000..02fd347 --- /dev/null +++ b/v3/scripts/main/phase4/experiment_grid.json @@ -0,0 +1,77 @@ +{ + "_notes": [ + "Phase 4 — Dual CNN architecture comparison (image only).", + "Goal: isolate the effect of bilateral processing by comparing architectures without clinical data.", + "Best settings from phase 3 carried forward: iop_ratio_drop_raw, bcd_p05 (p07 did not stack).", + "common_args are prepended to every run's args list.", + "All runs use --bridge-mode image_only — no clinical data." + ], + + "common_args": [ + "--eval-mode", "binary", + "--bridge-mode", "image_only", + "--epochs", "30", + "--in-memory-cache", + "--augment", + "--tune-binary-threshold", + "--backbone", "refugelike", + "--iop-corr-method", "ratio", + "--iop-drop-raw", + "--output-root", "v3/results" + ], + + "_common_args_implicit_defaults": { + "--tower-loss-mode": "bcd", + "--bcd-prob": "0.5", + "--warmup-cd-epochs": "0", + "--bilat-warmup-tower-epochs": "3", + "--bilat-warmup-fused-epochs": "3" + }, + + "baseline": { + "run_name": "phase4/single", + "description": "Single-eye baseline with best phase 3 image settings. Direct comparison point for bilateral modes.", + "extra_args": ["--tower-mode", "single"] + }, + + "groups": [ + { + "name": "architecture", + "description": "Core bilateral architecture comparison.", + "runs": [ + { + "run_name": "phase4/ensemble", + "description": "Ensemble: two independent single-eye forward passes, patient-level average of OD+OS scores.", + "extra_args": ["--tower-mode", "ensemble"] + }, + { + "run_name": "phase4/bilateral", + "description": "BilateralHT: shared towers, concat OD+OS → learned joint projection MLP → classifier.", + "extra_args": ["--tower-mode", "bilateral"] + }, + { + "run_name": "phase4/siamese", + "description": "SiameseHT: shared backbone, mean+delta (asymmetry) representation → classifier.", + "extra_args": ["--tower-mode", "siamese"] + } + ] + }, + + { + "name": "loss_bilateral", + "description": "Test loss function sensitivity in bilateral modes (using winner from architecture group).", + "runs": [ + { + "run_name": "phase4/bilateral_loss_all", + "description": "BilateralHT with all-losses mode — joint training dynamics may differ from single-eye.", + "extra_args": ["--tower-mode", "bilateral", "--tower-loss-mode", "all"] + }, + { + "run_name": "phase4/siamese_loss_all", + "description": "SiameseHT with all-losses mode.", + "extra_args": ["--tower-mode", "siamese", "--tower-loss-mode", "all"] + } + ] + } + ] +} diff --git a/v3/scripts/main/phase5/__init__.py b/v3/scripts/main/phase5/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/v3/scripts/main/phase5/dispatch_logit_mlp_ckpt.py b/v3/scripts/main/phase5/dispatch_logit_mlp_ckpt.py new file mode 100644 index 0000000..27609b9 --- /dev/null +++ b/v3/scripts/main/phase5/dispatch_logit_mlp_ckpt.py @@ -0,0 +1,170 @@ +""" +Dispatch a 10×5 rep-CV of logit_mlp_head with --save-checkpoints. + +Results land in v3/results/phase5/logit_mlp_head_ckpt/{rep00..rep09}/binary/ensemble/ + +Usage: + # Dry run + python -m v3.scripts.main.phase5.dispatch_logit_mlp_ckpt --dry-run + + # Submit to server + python -m v3.scripts.main.phase5.dispatch_logit_mlp_ckpt \ + --server http://hades:8765 --token hypertower + + # Skip reps already done, re-queue only missing ones: + python -m v3.scripts.main.phase5.dispatch_logit_mlp_ckpt \ + --server http://hades:8765 --token hypertower +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +import requests + +sys.path.insert(0, str(Path(__file__).resolve().parents[4])) + +RUN_NAME = "phase5/logit_mlp_head_ckpt" +MODULE = "v3.scripts.main.run_cv" +OUTPUT_DIR = "v3/results" +RESULTS_ROOT = Path(__file__).resolve().parents[4] / "v3" / "results" +REP_SEED_START = 100 +REP_SEED_STEP = 100 +N_REPS = 10 + +RUN_ARGS = [ + "--eval-mode", "binary", + "--bridge-mode", "fused", + "--tower-mode", "ensemble", + "--fused-head", + "--head-type", "logit_mlp", + "--epochs", "30", + "--in-memory-cache", + "--augment", + "--tune-binary-threshold", + "--backbone", "refugelike", + "--iop-corr-method", "ratio", + "--iop-drop-raw", + "--exclude-cols", "Axial_Length", + "--output-root", "v3/results", + "--save-checkpoints", +] + + +# ── Completion check ────────────────────────────────────────────────────────── + +def _completed_reps(reps: int) -> list[int]: + done = [] + for i in range(reps): + rep_dir = RESULTS_ROOT / RUN_NAME / f"rep{i:02d}" / "binary" / "ensemble" + if (rep_dir / "summary.json").exists(): + # Also verify at least one checkpoint exists + if any(rep_dir.glob("fold*/best_single.pt")): + done.append(i) + else: + print(f" [warn] rep{i:02d} has summary.json but no checkpoints — will re-queue") + return done + + +# ── Server API ──────────────────────────────────────────────────────────────── + +class _API: + def __init__(self, base_url: str, token: str): + self.base_url = base_url.rstrip("/") + self._h = {"x-token": token} + + def get(self, path: str, **params) -> object: + r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10) + r.raise_for_status() + return r.json() + + def post(self, path: str, body: dict) -> dict: + r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10) + r.raise_for_status() + return r.json() + + +def _queued_reps(jobs: list[dict]) -> set[int]: + active = set() + for job in jobs: + if job.get("run_name") != RUN_NAME: + continue + if job["state"] not in ("pending", "running"): + continue + try: + args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"]) + if "--rep-index" in args: + active.add(int(args[args.index("--rep-index") + 1])) + except Exception: + pass + return active + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--server", default=os.environ.get("HT_SERVER", "")) + ap.add_argument("--token", default=os.environ.get("HT_TOKEN", "")) + ap.add_argument("--reps", type=int, default=N_REPS) + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + + if not args.dry_run: + if not args.server: + ap.error("--server required (or set HT_SERVER)") + if not args.token: + ap.error("--token required (or set HT_TOKEN)") + + api = _API(args.server, args.token) if (args.server and args.token) else None + + server_jobs: list[dict] = [] + if api: + try: + all_jobs = api.get("/jobs") + server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")] + print(f"[server] {len(server_jobs)} job(s) pending/running") + except Exception as e: + print(f"[warn] could not fetch queue: {e}") + + done = set(_completed_reps(args.reps)) + queued = _queued_reps(server_jobs) + missing = [i for i in range(args.reps) if i not in (done | queued)] + + print(f"\nRun: {RUN_NAME}") + print(f" Done: {sorted(done)}") + print(f" Queued: {sorted(queued - done)}") + print(f" Missing: {missing}") + + if not missing: + print("Nothing to submit.") + return + + for i in missing: + seed = REP_SEED_START + i * REP_SEED_STEP + rep_args = RUN_ARGS + [ + "--run-name", RUN_NAME, + "--reps", "1", + "--rep-seed-start", str(seed), + "--rep-index", str(i), + ] + body = { + "run_name": RUN_NAME, + "module": MODULE, + "args": rep_args, + "output_dir": OUTPUT_DIR, + "priority": 0, + } + if args.dry_run: + print(f" [dry-run] rep{i:02d} seed={seed}") + else: + resp = api.post("/jobs", body) + print(f" queued rep{i:02d} seed={seed} job_id={resp['job_id']}") + + +if __name__ == "__main__": + main() diff --git a/v3/scripts/main/phase5/dispatch_phase5.py b/v3/scripts/main/phase5/dispatch_phase5.py new file mode 100644 index 0000000..c14907f --- /dev/null +++ b/v3/scripts/main/phase5/dispatch_phase5.py @@ -0,0 +1,199 @@ +""" +Dispatch phase 5 experiment runs to the distributed job server. + +Reads experiment_grid.json, checks which runs already have complete 10x5 results, +and submits the rest. Skips runs marked needs_implementation. + +Usage: + python -m v3.scripts.main.phase5.dispatch_phase5 \ + --server http://hades:8765 --token hypertower + + # Dry run (print what would be submitted, don't actually submit): + python -m v3.scripts.main.phase5.dispatch_phase5 \ + --server http://hades:8765 --token hypertower --dry-run + + # Override number of reps (default 10): + python -m v3.scripts.main.phase5.dispatch_phase5 \ + --server http://hades:8765 --token hypertower --reps 4 +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +import requests + +# Allow running as `python v3/scripts/main/phase3/dispatch_phase3.py` +sys.path.insert(0, str(Path(__file__).resolve().parents[4])) + +GRID_PATH = Path(__file__).parent / "experiment_grid.json" +RESULTS_ROOT = Path(__file__).resolve().parents[4] / "v3" / "results" +MODULE = "v3.scripts.main.run_cv" +OUTPUT_DIR = "v3/results" +REP_SEED_START = 100 +REP_SEED_STEP = 100 + + +# ── Completion check ────────────────────────────────────────────────────────── + +def _completed_reps(run_name: str, reps: int) -> list[int]: + """Return list of rep indices that already have a summary.json (any tower mode).""" + done = [] + for i in range(reps): + rep_dir = RESULTS_ROOT / run_name / f"rep{i:02d}" / "binary" + # Accept any tower mode subdir + if rep_dir.exists() and any((rep_dir / tm / "summary.json").exists() + for tm in ("single", "bilateral", "siamese", "ensemble")): + done.append(i) + return done + + +# ── Server API ──────────────────────────────────────────────────────────────── + +class _API: + def __init__(self, base_url: str, token: str): + self.base_url = base_url.rstrip("/") + self._h = {"x-token": token} + + def get(self, path: str, **params) -> object: + r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10) + r.raise_for_status() + return r.json() + + def post(self, path: str, body: dict) -> dict: + r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10) + r.raise_for_status() + return r.json() + + +def _queued_reps(jobs: list[dict], run_name: str) -> set[int]: + """Return rep indices already pending or running in the server queue.""" + active = set() + for job in jobs: + if job["run_name"] != run_name: + continue + if job["state"] not in ("pending", "running"): + continue + # Extract --rep-index from job args + try: + args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"]) + if "--rep-index" in args: + active.add(int(args[args.index("--rep-index") + 1])) + except Exception: + pass + return active + + +def _submit_cv(api: _API, run_name: str, run_args: list[str], + reps: int, missing: list[int], dry_run: bool): + """Submit one job per missing rep.""" + for i in missing: + seed = REP_SEED_START + i * REP_SEED_STEP + rep_args = run_args + [ + "--run-name", run_name, + "--reps", "1", + "--rep-seed-start", str(seed), + "--rep-index", str(i), + ] + body = { + "run_name": run_name, + "module": MODULE, + "args": rep_args, + "output_dir": OUTPUT_DIR, + "priority": 0, + } + if dry_run: + print(f" [dry-run] would queue rep{i:02d} seed={seed}") + else: + resp = api.post("/jobs", body) + print(f" queued rep{i:02d} seed={seed} job_id={resp['job_id']}") + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""), + help="Server URL (or set HT_SERVER)") + ap.add_argument("--token", default=os.environ.get("HT_TOKEN", ""), + help="Shared secret (or set HT_TOKEN)") + ap.add_argument("--reps", type=int, default=10, + help="Expected number of reps per run (default: 10)") + ap.add_argument("--grid", type=Path, default=GRID_PATH, + help="Path to experiment grid JSON (default: experiment_grid.json)") + ap.add_argument("--dry-run", action="store_true", + help="Print what would be submitted without actually submitting") + args = ap.parse_args() + + if not args.dry_run: + if not args.server: + ap.error("--server is required (or set HT_SERVER)") + if not args.token: + ap.error("--token is required (or set HT_TOKEN)") + elif not args.server or not args.token: + print("[dry-run] no --server/--token provided — skipping queue check, showing disk state only") + + grid = json.loads(args.grid.read_text()) + common_args = grid["common_args"] + api = _API(args.server, args.token) if (args.server and args.token) else None + + # Fetch current server queue once (pending + running) + server_jobs: list[dict] = [] + if api: + try: + all_jobs = api.get("/jobs") + server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")] + print(f"[server] {len(server_jobs)} job(s) currently pending/running in queue") + except Exception as e: + print(f"[warn] could not fetch server queue: {e}") + + # Collect all runs: baseline + every group's runs + all_runs = [grid["baseline"]] + for group in grid["groups"]: + if group.get("needs_implementation"): + print(f"\n[skip] group '{group['name']}' — {group['needs_implementation']}") + continue + all_runs.extend(group["runs"]) + + submitted_total = 0 + skipped_total = 0 + + for run in all_runs: + run_name = run["run_name"] + run_args = common_args + run.get("extra_args", []) + done = set(_completed_reps(run_name, args.reps)) + queued = _queued_reps(server_jobs, run_name) + accounted = done | queued + missing = [i for i in range(args.reps) if i not in accounted] + + if not missing: + if len(done) == args.reps: + print(f"\n[done] {run_name} ({args.reps}/{args.reps} reps complete)") + else: + in_q = sorted(queued - done) + print(f"\n[skip] {run_name} ({len(done)} done, {len(in_q)} queued: {[f'rep{i:02d}' for i in in_q]})") + skipped_total += 1 + continue + + parts = [] + if done: parts.append(f"{len(done)} done") + if queued: parts.append(f"{len(queued - done)} queued") + status = ", ".join(parts) if parts else "not started" + print(f"\n[queue] {run_name} ({status}) — submitting {len(missing)} rep(s)") + _submit_cv(api, run_name, run_args, args.reps, missing, args.dry_run) + submitted_total += len(missing) + + print(f"\n{'='*50}") + print(f"Submitted: {submitted_total} jobs | Already accounted for: {skipped_total} runs") + if grid.get("groups"): + needs_impl = sum(1 for g in grid["groups"] if g.get("needs_implementation")) + if needs_impl: + print(f"Skipped (needs implementation): {needs_impl} group(s)") + + +if __name__ == "__main__": + main() diff --git a/v3/scripts/main/phase5/experiment_grid.json b/v3/scripts/main/phase5/experiment_grid.json new file mode 100644 index 0000000..8338234 --- /dev/null +++ b/v3/scripts/main/phase5/experiment_grid.json @@ -0,0 +1,87 @@ +{ + "_notes": [ + "Phase 5 — Full HyperTower: bilateral + clinical data + aggregation strategy comparison.", + "Goal: show the effect of a dual CNN + clinical data, and compare ensemble vs fused-head.", + "Best settings from all prior phases: refugelike, iop_ratio_drop_raw, bcd_p05.", + "Best bilateral architecture from phase 4 should be used — update tower-mode accordingly.", + "common_args are prepended to every run's args list.", + "NOTE: update --tower-mode in groups below once phase 4 winner is known.", + "Placeholder uses 'bilateral' — change to 'siamese' if that wins phase 4." + ], + + "common_args": [ + "--eval-mode", "binary", + "--bridge-mode", "fused", + "--epochs", "30", + "--in-memory-cache", + "--augment", + "--tune-binary-threshold", + "--backbone", "refugelike", + "--iop-corr-method", "ratio", + "--iop-drop-raw", + "--exclude-cols", "Axial_Length", + "--output-root", "v3/results" + ], + + "_common_args_implicit_defaults": { + "--tower-loss-mode": "bcd", + "--bcd-prob": "0.5", + "--warmup-cd-epochs": "40", + "--single-warmup-tower-epochs": "3", + "--single-warmup-fused-epochs": "3", + "--bilat-warmup-tower-epochs": "3", + "--bilat-warmup-fused-epochs": "3" + }, + + "baseline": { + "run_name": "phase5/single_fused", + "description": "Single-eye + clinical data — phase 3 best config, re-run as direct comparison baseline for phase 5.", + "extra_args": ["--tower-mode", "single"] + }, + + "groups": [ + { + "name": "bilateral_clinical", + "description": "Add clinical data to bilateral architectures.", + "runs": [ + { + "run_name": "phase5/ensemble_fused", + "description": "Ensemble (independent OD+OS) + clinical data via fused bridge.", + "extra_args": ["--tower-mode", "ensemble"] + }, + { + "run_name": "phase5/bilateral_fused", + "description": "BilateralHT + clinical data — full canonical HyperTower.", + "extra_args": ["--tower-mode", "bilateral"] + }, + { + "run_name": "phase5/siamese_fused", + "description": "SiameseHT + clinical data — siamese mean+delta with fused clinical bridge.", + "extra_args": ["--tower-mode", "siamese"] + } + ] + }, + + { + "name": "aggregation", + "description": "Compare patient-level prediction aggregation strategies on top of ensemble.", + "runs": [ + { + "run_name": "phase5/ensemble_fused_head", + "description": "Ensemble + clinical data + attention scorer head (Linear(C→1) per eye, softmax-weighted average).", + "extra_args": ["--tower-mode", "ensemble", "--fused-head"] + }, + { + "run_name": "phase5/logit_mlp_head", + "description": "Ensemble + clinical data + logit-level MLP head (cat([logit_od, logit_os]) → FC(64) → FC(C)).", + "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--head-type", "logit_mlp"] + }, + { + "run_name": "phase5/embedding_mlp_head", + "description": "Ensemble + clinical data + embedding-level MLP head (cat([z_od, z_os]) → FC(256) → FC(C)).", + "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--head-type", "embedding_mlp"] + } + ] + } + ] +} diff --git a/v3/scripts/main/phase6/__init__.py b/v3/scripts/main/phase6/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/v3/scripts/main/phase6/dispatch_phase6a.py b/v3/scripts/main/phase6/dispatch_phase6a.py new file mode 100644 index 0000000..ba08c87 --- /dev/null +++ b/v3/scripts/main/phase6/dispatch_phase6a.py @@ -0,0 +1,204 @@ +""" +Dispatch phase 6a experiment runs (geometry vector injection) to the distributed job server. + +Covers the geometry_vector_gt and geometry_vector_unet groups from experiment_grid.json: + - GT geometry vector × {single, ensemble, fused-head} + - U-Net geometry vector × {single, ensemble, fused-head} + +Skips geometry_tower_* groups (needs_implementation — will get dispatch_phase6b.py). + +Usage: + python -m v3.scripts.main.phase6.dispatch_phase6a \ + --server http://hades:8765 --token hypertower + + # Dry run (print what would be submitted, don't actually submit): + python -m v3.scripts.main.phase6.dispatch_phase6a \ + --server http://hades:8765 --token hypertower --dry-run + + # Override number of reps (default 10): + python -m v3.scripts.main.phase6.dispatch_phase6a \ + --server http://hades:8765 --token hypertower --reps 4 + +NOTE: Requires --geometry-dim and --geometry-source to be wired into +v3_hypertower.py before these jobs will run successfully. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +import requests + +# Allow running as `python v3/scripts/main/phase6/dispatch_phase6a.py` +sys.path.insert(0, str(Path(__file__).resolve().parents[4])) + +GRID_PATH = Path(__file__).parent / "experiment_grid.json" +RESULTS_ROOT = Path(__file__).resolve().parents[4] / "v3" / "results" +MODULE = "v3.scripts.main.run_cv" +OUTPUT_DIR = "v3/results" +REP_SEED_START = 100 +REP_SEED_STEP = 100 + + +# ── Completion check ────────────────────────────────────────────────────────── + +def _completed_reps(run_name: str, reps: int) -> list[int]: + """Return list of rep indices that already have a summary.json (any tower mode).""" + done = [] + for i in range(reps): + rep_dir = RESULTS_ROOT / run_name / f"rep{i:02d}" / "binary" + # Accept any tower mode subdir + if rep_dir.exists() and any((rep_dir / tm / "summary.json").exists() + for tm in ("single", "bilateral", "siamese", "ensemble", "tri", "tri_bilateral")): + done.append(i) + return done + + +# ── Server API ──────────────────────────────────────────────────────────────── + +class _API: + def __init__(self, base_url: str, token: str): + self.base_url = base_url.rstrip("/") + self._h = {"x-token": token} + + def get(self, path: str, **params) -> object: + r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10) + r.raise_for_status() + return r.json() + + def post(self, path: str, body: dict) -> dict: + r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10) + r.raise_for_status() + return r.json() + + +def _queued_reps(jobs: list[dict], run_name: str) -> set[int]: + """Return rep indices already pending or running in the server queue.""" + active = set() + for job in jobs: + if job["run_name"] != run_name: + continue + if job["state"] not in ("pending", "running"): + continue + try: + args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"]) + if "--rep-index" in args: + active.add(int(args[args.index("--rep-index") + 1])) + except Exception: + pass + return active + + +def _submit_cv(api: _API, run_name: str, run_args: list[str], + reps: int, missing: list[int], dry_run: bool): + """Submit one job per missing rep.""" + for i in missing: + seed = REP_SEED_START + i * REP_SEED_STEP + rep_args = run_args + [ + "--run-name", run_name, + "--reps", "1", + "--rep-seed-start", str(seed), + "--rep-index", str(i), + ] + body = { + "run_name": run_name, + "module": MODULE, + "args": rep_args, + "output_dir": OUTPUT_DIR, + "priority": 0, + } + if dry_run: + print(f" [dry-run] would queue rep{i:02d} seed={seed}") + else: + resp = api.post("/jobs", body) + print(f" queued rep{i:02d} seed={seed} job_id={resp['job_id']}") + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""), + help="Server URL (or set HT_SERVER)") + ap.add_argument("--token", default=os.environ.get("HT_TOKEN", ""), + help="Shared secret (or set HT_TOKEN)") + ap.add_argument("--reps", type=int, default=10, + help="Expected number of reps per run (default: 10)") + ap.add_argument("--grid", type=Path, default=GRID_PATH, + help="Path to experiment grid JSON (default: experiment_grid.json)") + ap.add_argument("--dry-run", action="store_true", + help="Print what would be submitted without actually submitting") + args = ap.parse_args() + + if not args.dry_run: + if not args.server: + ap.error("--server is required (or set HT_SERVER)") + if not args.token: + ap.error("--token is required (or set HT_TOKEN)") + elif not args.server or not args.token: + print("[dry-run] no --server/--token provided — skipping queue check, showing disk state only") + + grid = json.loads(args.grid.read_text()) + common_args = grid["common_args"] + api = _API(args.server, args.token) if (args.server and args.token) else None + + # Fetch current server queue once (pending + running) + server_jobs: list[dict] = [] + if api: + try: + all_jobs = api.get("/jobs") + server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")] + print(f"[server] {len(server_jobs)} job(s) currently pending/running in queue") + except Exception as e: + print(f"[warn] could not fetch server queue: {e}") + + # Collect all runs: baseline + every group's runs (skip needs_implementation) + all_runs = [grid["baseline"]] + for group in grid["groups"]: + if group.get("needs_implementation"): + print(f"\n[skip] group '{group['name']}' — {group['needs_implementation']}") + continue + all_runs.extend(group["runs"]) + + submitted_total = 0 + skipped_total = 0 + + for run in all_runs: + run_name = run["run_name"] + run_args = common_args + run.get("extra_args", []) + done = set(_completed_reps(run_name, args.reps)) + queued = _queued_reps(server_jobs, run_name) + accounted = done | queued + missing = [i for i in range(args.reps) if i not in accounted] + + if not missing: + if len(done) == args.reps: + print(f"\n[done] {run_name} ({args.reps}/{args.reps} reps complete)") + else: + in_q = sorted(queued - done) + print(f"\n[skip] {run_name} ({len(done)} done, {len(in_q)} queued: {[f'rep{i:02d}' for i in in_q]})") + skipped_total += 1 + continue + + parts = [] + if done: parts.append(f"{len(done)} done") + if queued: parts.append(f"{len(queued - done)} queued") + status = ", ".join(parts) if parts else "not started" + print(f"\n[queue] {run_name} ({status}) — submitting {len(missing)} rep(s)") + _submit_cv(api, run_name, run_args, args.reps, missing, args.dry_run) + submitted_total += len(missing) + + print(f"\n{'='*50}") + print(f"Submitted: {submitted_total} jobs | Already accounted for: {skipped_total} runs") + if grid.get("groups"): + needs_impl = sum(1 for g in grid["groups"] if g.get("needs_implementation")) + if needs_impl: + print(f"Skipped (needs implementation): {needs_impl} group(s)") + + +if __name__ == "__main__": + main() diff --git a/v3/scripts/main/phase6/experiment_grid.json b/v3/scripts/main/phase6/experiment_grid.json new file mode 100644 index 0000000..fb0dab1 --- /dev/null +++ b/v3/scripts/main/phase6/experiment_grid.json @@ -0,0 +1,144 @@ +{ + "_notes": [ + "Phase 6 — Geometry augmentation: vector injection and dedicated geometry tower.", + "Goal: test whether derived structural geometry (CDR, rim ratio, etc.) improves performance", + " injected either as a 5-dim vector appended to the clinical stream (Part A)", + " or as a dedicated geometry tower feeding into bridge fusion (Part B).", + "Tower modes under test: single, ensemble, fused-head (ensemble + --fused-head).", + "Geometry sources: GT contour annotations (no annotator-bias concern at this stage —", + " labels were assigned by same clinicians, GT seg merely measures CDR directly)", + " and U-Net segmentations (REFUGE-trained, fine-tuned on PAPILA folds — unbiased).", + "Part A (geometry_vector): vector injection — dispatch_phase6a.py covers this.", + " Requires: --geometry-dim and --geometry-source wired into v3_hypertower.py.", + "Part B (geometry_tower): dedicated geometry tower — needs architecture implementation.", + " Will get its own dispatch_phase6b.py once built.", + "Ensemble and fused-head baselines come from phase5. Single has no equivalent with all tuned", + " hyperparameters, so a phase6 single baseline is included here.", + "Best settings from all prior phases: refugelike backbone, iop_ratio_drop_raw, bcd_p05.", + "Geometry features: [area_cdr, rim_ratio, vertical_cdr, horizontal_cdr, centre_shift] (dim=5).", + "common_args are prepended to every run's args list." + ], + + "common_args": [ + "--eval-mode", "binary", + "--bridge-mode", "fused", + "--epochs", "30", + "--in-memory-cache", + "--augment", + "--tune-binary-threshold", + "--backbone", "refugelike", + "--iop-corr-method", "ratio", + "--iop-drop-raw", + "--exclude-cols", "Axial_Length", + "--img-crop-manifest", "manifest.csv", + "--output-root", "v3/results" + ], + + "_common_args_implicit_defaults": { + "--tower-loss-mode": "bcd", + "--bcd-prob": "0.5", + "--warmup-cd-epochs": "40", + "--single-warmup-tower-epochs": "3", + "--single-warmup-fused-epochs": "3", + "--bilat-warmup-tower-epochs": "3", + "--bilat-warmup-fused-epochs": "3" + }, + + "baseline": { + "run_name": "phase6/single_no_geom", + "description": "Single-eye + clinical data, no geometry — needed as Phase 6 baseline since no prior phase ran single with all tuned hyperparameters (iop_ratio_drop_raw, bcd_p05, refugelike).", + "extra_args": ["--tower-mode", "single"] + }, + + "groups": [ + { + "name": "geometry_vector_gt", + "description": "Part A — Inject 5-dim geometry vector from GT annotations alongside clinical data. Three aggregation modes: single-eye, ensemble (independent OD+OS), ensemble with fused head.", + "runs": [ + { + "run_name": "phase6/vec_gt_single", + "description": "Single-eye + clinical + GT geometry vector appended to clinical stream.", + "extra_args": ["--tower-mode", "single", "--geometry-dim", "5", "--geometry-source", "gt"] + }, + { + "run_name": "phase6/vec_gt_ensemble", + "description": "Ensemble + clinical + GT geometry vector (per-eye geometry, independent OD+OS mean).", + "extra_args": ["--tower-mode", "ensemble", "--geometry-dim", "5", "--geometry-source", "gt"] + }, + { + "run_name": "phase6/vec_gt_fused_head", + "description": "Ensemble + fused head + clinical + GT geometry vector.", + "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--geometry-dim", "5", "--geometry-source", "gt"] + } + ] + }, + + { + "name": "geometry_vector_unet", + "description": "Part A — Same three modes but geometry from U-Net segmentations (REFUGE-trained, fine-tuned). Tests whether GT annotator bias affects the geometry signal.", + "runs": [ + { + "run_name": "phase6/vec_unet_single", + "description": "Single-eye + clinical + U-Net geometry vector.", + "extra_args": ["--tower-mode", "single", "--geometry-dim", "5", "--geometry-source", "unet"] + }, + { + "run_name": "phase6/vec_unet_ensemble", + "description": "Ensemble + clinical + U-Net geometry vector.", + "extra_args": ["--tower-mode", "ensemble", "--geometry-dim", "5", "--geometry-source", "unet"] + }, + { + "run_name": "phase6/vec_unet_fused_head", + "description": "Ensemble + fused head + clinical + U-Net geometry vector.", + "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--geometry-dim", "5", "--geometry-source", "unet"] + } + ] + }, + + { + "name": "geometry_tower_gt", + "description": "Part B — Dedicated geometry tower (MLP on 5-dim vector, separate from clinical tower) with GT geometry. Requires tri-tower bridge architecture.", + "needs_implementation": "Geometry tower not yet built — requires GeometryTower MLP, bridge reconfiguration for n>=3 towers, and --geometry-tower CLI flag.", + "runs": [ + { + "run_name": "phase6/tower_gt_single", + "description": "Single-eye + clinical tower + dedicated GT geometry tower.", + "extra_args": ["--tower-mode", "single", "--geometry-tower", "--geometry-source", "gt"] + }, + { + "run_name": "phase6/tower_gt_ensemble", + "description": "Ensemble + clinical tower + dedicated GT geometry tower.", + "extra_args": ["--tower-mode", "ensemble", "--geometry-tower", "--geometry-source", "gt"] + }, + { + "run_name": "phase6/tower_gt_fused_head", + "description": "Ensemble + fused head + clinical tower + dedicated GT geometry tower.", + "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--geometry-tower", "--geometry-source", "gt"] + } + ] + }, + + { + "name": "geometry_tower_unet", + "description": "Part B — Same three modes with dedicated geometry tower, U-Net geometry source.", + "needs_implementation": "Geometry tower not yet built — requires GeometryTower MLP, bridge reconfiguration for n>=3 towers, and --geometry-tower CLI flag.", + "runs": [ + { + "run_name": "phase6/tower_unet_single", + "description": "Single-eye + clinical tower + dedicated U-Net geometry tower.", + "extra_args": ["--tower-mode", "single", "--geometry-tower", "--geometry-source", "unet"] + }, + { + "run_name": "phase6/tower_unet_ensemble", + "description": "Ensemble + clinical tower + dedicated U-Net geometry tower.", + "extra_args": ["--tower-mode", "ensemble", "--geometry-tower", "--geometry-source", "unet"] + }, + { + "run_name": "phase6/tower_unet_fused_head", + "description": "Ensemble + fused head + clinical tower + dedicated U-Net geometry tower.", + "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--geometry-tower", "--geometry-source", "unet"] + } + ] + } + ] +} diff --git a/v3/scripts/main/run_cv.py b/v3/scripts/main/run_cv.py index 8f11a76..f9811ec 100644 --- a/v3/scripts/main/run_cv.py +++ b/v3/scripts/main/run_cv.py @@ -54,6 +54,11 @@ def build_parser() -> argparse.ArgumentParser: "--rep-seed-step", type=int, default=100, help="Increment between rep fold-seeds (default: 100; rep k uses seed start + k*step).", ) + ap.add_argument( + "--rep-index", type=int, default=None, + help="Override the rep directory index (e.g. 3 → rep03). " + "Used by the distributed server to run a single rep of a multi-rep job.", + ) return ap @@ -65,22 +70,24 @@ def main(): seed_start = int(args.rep_seed_start) seed_step = int(args.rep_seed_step) base_run_name = args.run_name or "v3_cv" + rep_index_override = getattr(args, "rep_index", None) for rep in range(reps): rep_seed = seed_start + rep * seed_step args.fold_seed = rep_seed - if reps > 1: - args.run_name = f"{base_run_name}/rep{rep:02d}" + dir_index = rep_index_override if (rep_index_override is not None and reps == 1) else rep + if reps > 1 or rep_index_override is not None: + args.run_name = f"{base_run_name}/rep{dir_index:02d}" print(f"\n{'='*60}", flush=True) - print(f"Rep {rep+1}/{reps} fold_seed={rep_seed}", flush=True) + print(f"Rep {dir_index+1} fold_seed={rep_seed}", flush=True) print(f"{'='*60}", flush=True) else: args.run_name = base_run_name tower = V3HyperTower(args) out_dir = tower.run() - print(f"\nRep {rep+1} output: {out_dir}", flush=True) + print(f"\nRep {dir_index+1} output: {out_dir}", flush=True) if __name__ == "__main__": diff --git a/v3/scripts/output_analysis/analyze_phases.py b/v3/scripts/output_analysis/analyze_phases.py new file mode 100644 index 0000000..8be2652 --- /dev/null +++ b/v3/scripts/output_analysis/analyze_phases.py @@ -0,0 +1,809 @@ +""" +Phase 1–5 analysis — ablation plots + pairwise Wilcoxon tests. + +Usage: + python -m v3.scripts.output_analysis.analyze_phases # all phases + python -m v3.scripts.output_analysis.analyze_phases --phase 3 + python -m v3.scripts.output_analysis.analyze_phases --out figures/ +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from scipy.stats import wilcoxon +from sklearn.metrics import roc_auc_score +from statsmodels.stats.multitest import multipletests + +RESULTS_ROOT = Path(__file__).resolve().parents[3] / "v3" / "results" + +# Shared style constants +C_BASELINE = "#dd8452" +C_OTHER = "#4c72b0" +C_MEDIAN = "#c44e52" +FSIZE = 10 + + +# ── Helpers ────────────────────────────────────────────────────────────────── + +def load_rep_aucs(run_path: Path, test_key: str = "classic_test") -> np.ndarray: + """Return array of per-rep AUC means for a run directory.""" + aucs = [] + for rep_dir in sorted(run_path.glob("rep*")): + for summary in rep_dir.rglob("summary.json"): + txt = summary.read_text().strip() + if not txt: + continue + d = json.loads(txt) + auc = d.get("mode_summary", {}).get(test_key, {}).get("auc_mean") + if auc is not None: + aucs.append(auc) + break # one summary per rep + return np.array(aucs) + + +def wilcoxon_p(a: np.ndarray, b: np.ndarray) -> float: + """Two-sided Wilcoxon signed-rank p-value; returns nan if underpowered.""" + diffs = a - b + if np.all(diffs == 0) or len(diffs) < 5: + return float("nan") + try: + return wilcoxon(diffs, alternative="two-sided").pvalue + except Exception: + return float("nan") + + +def stars(p: float) -> str: + if np.isnan(p): return "" + if p < 0.001: return "***" + if p < 0.01: return "**" + if p < 0.05: return "*" + return "ns" + + +def paired_matrix(runs: list[str], aucs_dict: dict[str, np.ndarray], + fdr: bool = True) -> tuple[np.ndarray, np.ndarray]: + """Return (p_matrix, corrected_p_matrix) shape (n, n).""" + n = len(runs) + raw = np.full((n, n), np.nan) + for i, a in enumerate(runs): + for j, b in enumerate(runs): + if i != j and a in aucs_dict and b in aucs_dict: + ai, bi = aucs_dict[a], aucs_dict[b] + min_n = min(len(ai), len(bi)) + if min_n >= 5: + raw[i, j] = wilcoxon_p(ai[:min_n], bi[:min_n]) + if fdr: + mask = ~np.isnan(raw) + if mask.sum() > 0: + flat = raw[mask] + _, corrected, _, _ = multipletests(flat, method="fdr_bh") + corr = raw.copy() + corr[mask] = corrected + return raw, corr + return raw, raw.copy() + + +def boxplot_panel(ax, data_list, labels, base_idx, title="", ylabel="Test AUC", + base_aucs=None, all_aucs_by_label=None): + """Vertical box plot with p-value vs baseline under each tick label.""" + n = len(labels) + x = np.arange(n) + colors = [C_BASELINE if i == base_idx else C_OTHER for i in range(n)] + + bp = ax.boxplot(data_list, vert=True, patch_artist=True, positions=x, + widths=0.3, showfliers=True, + flierprops=dict(marker="o", markersize=3, alpha=0.5), + medianprops=dict(color=C_MEDIAN, linewidth=2)) + for patch, color in zip(bp["boxes"], colors): + patch.set_facecolor(color) + patch.set_alpha(0.8) + + if base_aucs is not None: + ax.axhline(np.median(base_aucs), color=C_BASELINE, linewidth=1, + linestyle="--", alpha=0.5, label="Baseline median") + ax.legend(fontsize=FSIZE - 1) + + ax.set_xlim(-0.5, n - 0.5) + ax.set_ylabel(ylabel, fontsize=FSIZE + 1) + if title: + ax.set_title(title, fontsize=FSIZE + 1, fontweight="bold") + ax.grid(axis="y", alpha=0.3) + + tick_labels = [] + for i, lbl in enumerate(labels): + if i == base_idx or base_aucs is None: + tick_labels.append(lbl) + continue + a = (all_aucs_by_label or {}).get(lbl, data_list[i]) + min_n = min(len(a), len(base_aucs)) + p = wilcoxon_p(a[:min_n], base_aucs[:min_n]) + p_str = f"p={p:.3f}" if not np.isnan(p) else "p=n/a" + tick_labels.append(f"{lbl}\n{p_str}") + + ax.set_xticks(x) + ax.set_xticklabels(tick_labels, fontsize=FSIZE) + + +def bar_plot(ax, labels, means, stds, baseline_idx, title, ylabel="AUC", + baseline_aucs=None, all_aucs=None): + """Horizontal bar chart with baseline highlighted and p-value annotations.""" + n = len(labels) + colors = ["#4c72b0" if i != baseline_idx else "#dd8452" for i in range(n)] + y = np.arange(n) + bars = ax.barh(y, means, xerr=stds, color=colors, alpha=0.85, + height=0.6, capsize=3, error_kw=dict(linewidth=1)) + ax.set_yticks(y) + ax.set_yticklabels(labels, fontsize=8) + ax.set_xlabel(ylabel) + ax.set_title(title, fontsize=10, fontweight="bold") + ax.axvline(means[baseline_idx], color="#dd8452", linewidth=1, linestyle="--", alpha=0.6) + + # Annotate with p-value stars vs baseline + if baseline_aucs is not None and all_aucs is not None: + x_max = max(means) + max(stds) + 0.005 + for i, lbl in enumerate(labels): + if i == baseline_idx: + continue + a = all_aucs.get(lbl) + if a is None: + continue + min_n = min(len(a), len(baseline_aucs)) + p = wilcoxon_p(a[:min_n], baseline_aucs[:min_n]) + s = stars(p) + if s: + ax.text(x_max, i, s, va="center", fontsize=7, + color="black" if s != "ns" else "gray") + + +def pairwise_heatmap(ax, runs, p_matrix, title): + """Lower-triangle heatmap of corrected p-values.""" + n = len(runs) + display = np.full_like(p_matrix, np.nan) + for i in range(n): + for j in range(i): + display[i, j] = p_matrix[i, j] + + im = ax.imshow(display, vmin=0, vmax=0.1, cmap="RdYlGn_r", aspect="auto") + ax.set_xticks(range(n)) + ax.set_yticks(range(n)) + ax.set_xticklabels(runs, rotation=45, ha="right", fontsize=7) + ax.set_yticklabels(runs, fontsize=7) + ax.set_title(title, fontsize=10, fontweight="bold") + plt.colorbar(im, ax=ax, label="p-value (FDR)") + + for i in range(n): + for j in range(i): + p = display[i, j] + if not np.isnan(p): + ax.text(j, i, f"{p:.2f}", ha="center", va="center", + fontsize=6, color="white" if p < 0.05 else "black") + + +# ── Phase 1 ────────────────────────────────────────────────────────────────── + +P1_CLF_ORDER = ["KNN", "Random Forest", "SVM", "Logistic Regression"] +P1_CLF_LABELS = {"KNN": "KNN", "Random Forest": "RF", "SVM": "SVM", "Logistic Regression": "LR"} +P1_TAGS = [ + ("no_leakage", "baseline"), + ("hypertower_loader", "HT loader"), +] +P1_BASELINE_TAG = "no_leakage" +P1_PAPER_AUC = {"KNN": 0.75, "Random Forest": 0.64, "SVM": 0.75, "Logistic Regression": 0.70} + +P1_BACKBONES = ["densenet121", "vgg16", "mobilenet_v2", "inception_v3", "resnet50"] +P1_BACKBONE_LABELS = { + "densenet121": "DenseNet121", + "vgg16": "VGG16", + "mobilenet_v2": "MobileNetV2", + "inception_v3": "InceptionV3", + "resnet50": "ResNet50", +} +P1_PAPER_CNN = { + "densenet121": (0.80, 0.05), "vgg16": (0.84, 0.02), + "mobilenet_v2": (0.75, 0.06), "inception_v3": (0.78, 0.08), "resnet50": (0.78, 0.07), +} + + +def _load_p1_clf_aucs(phase1_dir: Path) -> dict: + """Load per-fold AUCs for each tag × classifier.""" + data = {} + for tag, _ in P1_TAGS: + data[tag] = {} + for clf in P1_CLF_ORDER: + fpath = phase1_dir / tag / clf / "fold_metrics.csv" + if fpath.exists(): + data[tag][clf] = pd.read_csv(fpath)["auc"].tolist() + return data + + +def _load_p1_cnn_aucs(phase1_dir: Path) -> tuple[dict, dict]: + cnn, ht = {}, {} + for b in P1_BACKBONES: + fpath = phase1_dir / f"cnn_{b}" / "fold_metrics.csv" + cnn[b] = pd.read_csv(fpath)["auc"].tolist() if fpath.exists() else [] + aucs = [] + for fold in range(5): + yp = phase1_dir / "imageonly_ht" / b / "binary" / "single" / f"fold{fold}" / "test_y_true.npy" + pp = phase1_dir / "imageonly_ht" / b / "binary" / "single" / f"fold{fold}" / "test_probs_fused.npy" + if yp.exists() and pp.exists(): + y, pr = np.load(yp), np.load(pp) + if len(np.unique(y)) >= 2: + aucs.append(float(roc_auc_score(y, pr[:, 1]))) + ht[b] = aucs + return cnn, ht + + +def analyze_phase1(out_dir: Path): + print("\n=== Phase 1 ===") + phase1_dir = RESULTS_ROOT / "phase1" + + # ── Clinical classifiers ──────────────────────────────────────────────── + clf_data = _load_p1_clf_aucs(phase1_dir) + n_clf = len(P1_CLF_ORDER) + n_tags = len(P1_TAGS) + group_w = 0.7 + box_w = group_w / n_tags * 0.85 + offsets = np.linspace(-group_w / 2 + box_w / 2, group_w / 2 - box_w / 2, n_tags) + tag_colors = [C_BASELINE if t == P1_BASELINE_TAG else C_OTHER for t, _ in P1_TAGS] + + fig1, ax1 = plt.subplots(figsize=(10, 5)) + fig1.suptitle("Phase 1 — Clinical-only classifiers: CV strategy comparison", + fontsize=FSIZE + 2, fontweight="bold") + + for ti, (tag, lbl) in enumerate(P1_TAGS): + color = tag_colors[ti] + first = True + for ci, clf in enumerate(P1_CLF_ORDER): + aucs = clf_data.get(tag, {}).get(clf, []) + if not aucs: + continue + bp = ax1.boxplot(aucs, positions=[ci + offsets[ti]], widths=box_w, + patch_artist=True, manage_ticks=False, + boxprops=dict(facecolor=color, alpha=0.8), + medianprops=dict(color=C_MEDIAN, linewidth=2), + whiskerprops=dict(color=color, linewidth=1.2), + capprops=dict(color=color, linewidth=1.2), + flierprops=dict(marker="o", markersize=3, alpha=0.5)) + if first: + bp["boxes"][0].set_label(lbl) + first = False + + for ci, clf in enumerate(P1_CLF_ORDER): + if clf in P1_PAPER_AUC: + ax1.hlines(P1_PAPER_AUC[clf], ci - group_w / 2, ci + group_w / 2, + colors="black", linestyles=":", linewidths=1.5, + label="PAPILA paper" if ci == 0 else "_nolegend_") + + ax1.set_xticks(range(n_clf)) + ax1.set_xticklabels([P1_CLF_LABELS[c] for c in P1_CLF_ORDER], fontsize=FSIZE + 1) + ax1.set_ylabel("Test AUC", fontsize=FSIZE + 1) + ax1.set_ylim(0.45, 1.02) + ax1.axhline(0.5, color="grey", linestyle="--", linewidth=0.8, alpha=0.4) + ax1.grid(axis="y", alpha=0.3) + ax1.legend(fontsize=FSIZE, loc="lower right", framealpha=0.9) + fig1.tight_layout() + p1 = out_dir / "phase1_clinical_classifiers.png" + fig1.savefig(p1, dpi=150, bbox_inches="tight") + plt.close(fig1) + print(f" Saved: {p1}") + + # ── CNN backbones ─────────────────────────────────────────────────────── + cnn_data, ht_data = _load_p1_cnn_aucs(phase1_dir) + n_b = len(P1_BACKBONES) + offsets2 = [-group_w / 4, group_w / 4] + method_colors = [C_OTHER, C_BASELINE] + + fig2, ax2 = plt.subplots(figsize=(11, 5)) + fig2.suptitle("Phase 1 — CNN backbone: standalone vs HyperTower (image only)", + fontsize=FSIZE + 2, fontweight="bold") + + for bi, backbone in enumerate(P1_BACKBONES): + for si, (lbl, data, color) in enumerate([ + ("CNN standalone", cnn_data, method_colors[0]), + ("HyperTower", ht_data, method_colors[1]), + ]): + aucs = data.get(backbone, []) + if not aucs: + continue + bp = ax2.boxplot(aucs, positions=[bi + offsets2[si]], widths=box_w, + patch_artist=True, manage_ticks=False, + boxprops=dict(facecolor=color, alpha=0.8), + medianprops=dict(color=C_MEDIAN, linewidth=2), + whiskerprops=dict(color=color, linewidth=1.2), + capprops=dict(color=color, linewidth=1.2), + flierprops=dict(marker="o", markersize=3, alpha=0.5)) + if bi == 0: + bp["boxes"][0].set_label(lbl) + + if backbone in P1_PAPER_CNN: + mean_p, _ = P1_PAPER_CNN[backbone] + ax2.hlines(mean_p, bi - group_w / 2, bi + group_w / 2, + colors="black", linestyles=":", linewidths=1.5, + label="PAPILA paper" if bi == 0 else "_nolegend_") + + ax2.set_xticks(range(n_b)) + ax2.set_xticklabels([P1_BACKBONE_LABELS[b] for b in P1_BACKBONES], fontsize=FSIZE) + ax2.set_ylabel("Test AUC", fontsize=FSIZE + 1) + ax2.set_ylim(0.45, 1.02) + ax2.axhline(0.5, color="grey", linestyle="--", linewidth=0.8, alpha=0.4) + ax2.grid(axis="y", alpha=0.3) + ax2.legend(fontsize=FSIZE, loc="lower right", framealpha=0.9) + fig2.tight_layout() + p2 = out_dir / "phase1_cnn_backbones.png" + fig2.savefig(p2, dpi=150, bbox_inches="tight") + plt.close(fig2) + print(f" Saved: {p2}") + + +# ── Phase 2 ────────────────────────────────────────────────────────────────── + +PHASE2_RUNS = [ + ("imageonly_resnet50_leaky", "classic_test", "leaky CV"), + ("imageonly_resnet50_proper", "classic_test", "baseline"), + ("imageonly_refugelike_proper", "classic_test", "pretrained"), + ("imageonly_resnet50_gtcrop_1.1", "classic_test", "GT crop 1.1x"), + ("imageonly_resnet50_gtcrop_2.5", "classic_test", "GT crop 2.5x"), + ("imageonly_resnet50_unetcrop_1.1","classic_test", "UNet crop 1.1x"), + ("imageonly_resnet50_unetcrop_2.5","classic_test", "UNet crop 2.5x"), +] +PHASE2_BASELINE = "imageonly_resnet50_proper" +PHASE2_GROUPS = { + "Backbone": ["imageonly_resnet50_proper", "imageonly_refugelike_proper"], + "GT crop": ["imageonly_resnet50_proper", "imageonly_resnet50_gtcrop_1.1", "imageonly_resnet50_gtcrop_2.5"], + "UNet crop": ["imageonly_resnet50_proper", "imageonly_resnet50_unetcrop_1.1", "imageonly_resnet50_unetcrop_2.5"], + "Data leakage": ["imageonly_resnet50_leaky", "imageonly_resnet50_proper"], +} + + +def analyze_phase2(out_dir: Path): + print("\n=== Phase 2 ===") + aucs = {} + for run, key, _ in PHASE2_RUNS: + a = load_rep_aucs(RESULTS_ROOT / "phase2" / run, key) + aucs[run] = a + print(f" {run:40s} AUC={np.mean(a):.3f}±{np.std(a):.3f} n={len(a)}") + + # Leakage impact + leaky = aucs.get("imageonly_resnet50_leaky", np.array([])) + proper = aucs.get("imageonly_resnet50_proper", np.array([])) + if len(leaky) and len(proper): + min_n = min(len(leaky), len(proper)) + p = wilcoxon_p(leaky[:min_n], proper[:min_n]) + delta = np.mean(leaky) - np.mean(proper) + print(f"\n Data leakage inflates AUC by {delta:+.3f} (Wilcoxon p={p:.4f})") + + # Build display order: baseline first, then non-baseline sorted by mean AUC descending + base_run = PHASE2_BASELINE + base_label = next(lbl for r, _, lbl in PHASE2_RUNS if r == base_run) + others = [(r, lbl) for r, _, lbl in PHASE2_RUNS if r != base_run] + others.sort(key=lambda x: -np.mean(aucs[x[0]]) if len(aucs.get(x[0], [])) else float("inf")) + ordered = [(base_run, base_label)] + others + + run_names = [r for r, _ in ordered] + labels = [lbl for _, lbl in ordered] + base_idx = 0 + base_aucs = aucs[base_run] + + fig, ax = plt.subplots(figsize=(14, 6)) + fig.suptitle("Phase 2 — ResNet50: Backbone & Preprocessing Comparison", fontsize=12, fontweight="bold") + + data = [aucs[r] for r in run_names] + colors = ["#dd8452" if r == base_run else "#4c72b0" for r in run_names] + x = np.arange(len(run_names)) + + bp = ax.boxplot(data, vert=True, patch_artist=True, positions=x, + widths=0.3, showfliers=True, + flierprops=dict(marker="o", markersize=3, alpha=0.5), + medianprops=dict(color="#c44e52", linewidth=2)) + for patch, color in zip(bp["boxes"], colors): + patch.set_facecolor(color) + patch.set_alpha(0.8) + + ax.set_xlim(-0.5, len(run_names) - 0.5) + ax.set_ylabel("Test AUC", fontsize=11) + ax.axhline(np.median(base_aucs), color="#dd8452", linewidth=1, + linestyle="--", alpha=0.5, label="Baseline median") + ax.legend(fontsize=9) + ax.grid(axis="y", alpha=0.3) + + # Build x-tick labels with p-value on a second line underneath + tick_labels = [] + for i, run in enumerate(run_names): + if run == base_run: + tick_labels.append(labels[i]) + continue + a = aucs[run] + min_n = min(len(a), len(base_aucs)) + p = wilcoxon_p(a[:min_n], base_aucs[:min_n]) + p_str = f"p={p:.3f}" if not np.isnan(p) else "p=n/a" + tick_labels.append(f"{labels[i]}\n{p_str}") + + ax.set_xticks(x) + ax.set_xticklabels(tick_labels, fontsize=10) + + fig.tight_layout() + path = out_dir / "phase2_analysis.png" + fig.savefig(path, dpi=150, bbox_inches="tight") + plt.close(fig) + print(f" Saved: {path}") + + +# ── Phase 3 ────────────────────────────────────────────────────────────────── + +PHASE3_GROUPS = { + "Loss function": { + "baseline": "Baseline (BCE fused)", + "loss_all": "All losses", + "loss_bcd_p03":"BCD p=0.3", + "loss_bcd_p07":"BCD p=0.7", + }, + "SE attention": { + "baseline": "Baseline", + "se_img_tower": "SE img tower", + "se_cd_tower": "SE cd tower", + "se_bridge": "SE bridge", + "se_all": "SE all", + }, + "IOP correction": { + "baseline": "Baseline (none)", + "iop_ratio": "Ratio", + "iop_ratio_drop_raw":"Ratio + drop raw", + "iop_ols": "OLS", + "iop_lad": "LAD", + "iop_multi": "Multi", + }, + "Feature ablation": { + "baseline": "Baseline (all)", + "excl_iop": "Excl IOP", + "excl_age": "Excl age", + "excl_axial_length":"Excl axial length", + "excl_refractive": "Excl refractive", + }, + "Network dims": { + "baseline": "Baseline", + "cd_hidden_64": "CD hidden=64", + "cd_hidden_256": "CD hidden=256", + "fusion_dim_128": "Fusion dim=128", + "fusion_dim_512": "Fusion dim=512", + }, + "Dropout": { + "baseline": "Baseline (0.5)", + "bridge_dropout_03":"Bridge drop=0.3", + "bridge_dropout_07":"Bridge drop=0.7", + "cd_dropout_03": "CD drop=0.3", + }, + "Backbone freezing": { + "baseline": "Baseline (75%)", + "freeze_25": "Freeze 25%", + "freeze_50": "Freeze 50%", + }, + "Warmup": { + "baseline": "Baseline (cd40+twr3+fus3)", + "warmup_no_cd": "No CD warmup", + "warmup_tower5_fused5": "Tower5+Fused5", + }, + "Sampling": { + "baseline": "Baseline", + "balanced_sampling": "Balanced sampling", + }, + "Epoch length": { + "epochs_1": "1 epoch", + "epochs_5": "5 epochs", + "epochs_10": "10 epochs", + "epochs_20": "20 epochs", + "epochs_30": "30 epochs (baseline)", + "epochs_50": "50 epochs", + }, + "Learning rate": { + "baseline": "Baseline (1e-4)", + "lr_3e4": "3e-4", + "lr_1e3": "1e-3", + "lr_1e5": "1e-5", + }, +} +PHASE3_BASELINE = "baseline" + + +def analyze_phase3(out_dir: Path): + print("\n=== Phase 3 ===") + all_runs = set() + for group in PHASE3_GROUPS.values(): + all_runs.update(group.keys()) + aucs = {} + for run in all_runs: + a = load_rep_aucs(RESULTS_ROOT / "phase3" / run, "classic_test") + aucs[run] = a + base_aucs = aucs[PHASE3_BASELINE] + print(f" Baseline AUC: {np.mean(base_aucs):.3f}±{np.std(base_aucs):.3f}") + + # Build display order: baseline first, then each group (non-baseline, sorted desc) + GAP = 1.2 # extra space between groups + pos = 0.0 + positions, box_data, tick_labels, colors, is_sig = [], [], [], [], [] + group_spans = [] # (x_mid, group_name) for title annotations + + # Baseline box + positions.append(pos) + box_data.append(base_aucs) + tick_labels.append("baseline") + colors.append(C_BASELINE) + is_sig.append(False) + pos += 1 + GAP + + def _group_max_median(group_runs): + vals = [np.median(aucs[r]) for r in group_runs if r != PHASE3_BASELINE and len(aucs.get(r, []))] + return max(vals) if vals else 0.0 + + sorted_groups = sorted(PHASE3_GROUPS.items(), key=lambda x: -_group_max_median(x[1])) + + for shade_idx, (group_name, group_runs) in enumerate(sorted_groups): + non_base = [(r, lbl) for r, lbl in group_runs.items() if r != PHASE3_BASELINE] + non_base.sort(key=lambda x: -np.mean(aucs[x[0]]) if len(aucs.get(x[0], [])) else float("inf")) + + group_start = pos + for run, lbl in non_base: + a = aucs.get(run, np.array([])) + positions.append(pos) + box_data.append(a) + # p-value label under name + min_n = min(len(a), len(base_aucs)) + p = wilcoxon_p(a[:min_n], base_aucs[:min_n]) if min_n >= 5 else float("nan") + sig = not np.isnan(p) and p < 0.05 + if sig: + tick_labels.append(f"* {lbl}\np={p:.3f}") + else: + tick_labels.append(lbl) + colors.append(C_OTHER) + is_sig.append(sig) + pos += 1 + group_spans.append(((group_start + pos - 1) / 2, group_name, group_start, pos - 1, shade_idx)) + pos += GAP + + fig, ax = plt.subplots(figsize=(9, 22)) + fig.suptitle("Phase 3 — Clinical Fusion Ablations (Single-Eye)", + fontsize=FSIZE + 3, fontweight="bold") + fig.subplots_adjust(top=0.97, left=0.38) + + bp = ax.boxplot(box_data, vert=False, patch_artist=True, positions=positions, + widths=0.5, showfliers=True, + flierprops=dict(marker="o", markersize=3, alpha=0.5), + medianprops=dict(color=C_MEDIAN, linewidth=2), + manage_ticks=False) + for patch, color in zip(bp["boxes"], colors): + patch.set_facecolor(color) + patch.set_alpha(0.8) + + # Alternating shaded group backgrounds + for x_mid, gname, g_start, g_end, shade_idx in group_spans: + if shade_idx % 2 == 0: + ax.axhspan(g_start - 0.5, g_end + 0.5, color="gray", alpha=0.07, zorder=0) + # Group title to the left of the y-tick labels + ax.text(-0.42, x_mid, gname, transform=ax.get_yaxis_transform(), + ha="right", va="center", fontsize=FSIZE - 1, fontweight="bold", color="#444444") + + ax.axvline(np.median(base_aucs), color=C_BASELINE, linewidth=1, + linestyle="--", alpha=0.5, label="Baseline median") + ax.set_yticks(positions) + ax.set_yticklabels(tick_labels, fontsize=FSIZE - 1) + for tick_lbl, sig in zip(ax.get_yticklabels(), is_sig): + if sig: + tick_lbl.set_fontweight("bold") + ax.set_xlabel("Test AUC", fontsize=FSIZE + 1) + ax.set_xlim(0.65, None) + ax.set_ylim(-0.7, pos - GAP + 0.7) + ax.invert_yaxis() # baseline at top + ax.grid(axis="x", alpha=0.3) + ax.legend(fontsize=FSIZE, loc="lower right") + path = out_dir / "phase3_single_mode_ablations.png" + fig.savefig(path, dpi=150, bbox_inches="tight") + plt.close(fig) + print(f" Saved: {path}") + + # Print top winners vs baseline + print("\n Top movers vs baseline (Wilcoxon, uncorrected):") + deltas = [] + for run in aucs: + if run == PHASE3_BASELINE: + continue + a = aucs[run] + min_n = min(len(a), len(base_aucs)) + if min_n < 5: + continue + delta = np.mean(a) - np.mean(base_aucs) + p = wilcoxon_p(a[:min_n], base_aucs[:min_n]) + deltas.append((run, delta, p)) + deltas.sort(key=lambda x: -x[1]) + for run, delta, p in deltas[:8]: + print(f" {run:30s} {delta:+.3f} p={p:.4f} {stars(p)}") + + # Pairwise table — IOP correction group (FDR-corrected Wilcoxon p-values) + iop_runs = list(PHASE3_GROUPS["IOP correction"].keys()) + iop_labels = list(PHASE3_GROUPS["IOP correction"].values()) + _, corr = paired_matrix(iop_runs, aucs) + reports_dir = out_dir / "reports" + reports_dir.mkdir(exist_ok=True) + import csv + path2 = reports_dir / "phase3_iop_pairwise.csv" + with open(path2, "w", newline="") as f: + w = csv.writer(f) + w.writerow([""] + iop_labels) + for i, row_lbl in enumerate(iop_labels): + cells = [row_lbl] + for j in range(len(iop_labels)): + p = corr[i, j] + cells.append(f"{p:.4f} {stars(p)}" if not np.isnan(p) else "—") + w.writerow(cells) + print(f" Saved: {path2}") + + +# ── Phase 4 ────────────────────────────────────────────────────────────────── + +PHASE4_RUNS = { + "single": ("classic_test", "Single-eye\n(baseline)"), + "ensemble": ("ensemble_test", "Ensemble\n(indep OD+OS)"), + "bilateral": ("bilat_test", "BilateralHT\n(shared+concat)"), + "siamese": ("bilat_test", "SiameseHT\n(mean+delta)"), + "bilateral_loss_all": ("bilat_test", "BilateralHT\nall-losses"), + "siamese_loss_all": ("bilat_test", "SiameseHT\nall-losses"), +} +PHASE4_BASELINE = "single" + + +def analyze_phase4(out_dir: Path): + print("\n=== Phase 4 ===") + aucs = {} + for run, (key, _) in PHASE4_RUNS.items(): + a = load_rep_aucs(RESULTS_ROOT / "phase4" / run, key) + aucs[run] = a + print(f" {run:25s} AUC={np.mean(a):.3f}±{np.std(a):.3f} n={len(a)}") + + base_run = PHASE4_BASELINE + base_aucs = aucs[base_run] + others = [(r, PHASE4_RUNS[r][1]) for r in PHASE4_RUNS if r != base_run] + others.sort(key=lambda x: -np.mean(aucs[x[0]]) if len(aucs.get(x[0], [])) else float("inf")) + ordered = [(base_run, PHASE4_RUNS[base_run][1])] + others + run_keys = [r for r, _ in ordered] + labels = [lbl for _, lbl in ordered] + data_list = [aucs[r] for r in run_keys] + aucs_by_label = {lbl: aucs[r] for r, lbl in ordered} + + fig, ax = plt.subplots(figsize=(11, 5)) + fig.suptitle("Phase 4 — Bilateral Architecture Comparison (Image Only)", + fontsize=FSIZE + 2, fontweight="bold") + boxplot_panel(ax, data_list, labels, base_idx=0, + base_aucs=base_aucs, all_aucs_by_label=aucs_by_label) + ax.set_ylim(0.75, None) + + fig.tight_layout() + path = out_dir / "phase4_analysis.png" + fig.savefig(path, dpi=150, bbox_inches="tight") + plt.close(fig) + print(f" Saved: {path}") + + +# ── Phase 5 ────────────────────────────────────────────────────────────────── + +PHASE5_RUNS = { + # "single_fused": ("classic_test", "Single-eye (baseline)"), + "ensemble_fused": ("ensemble_test", "Ensemble (baseline)"), + "bilateral_fused": ("bilat_test", "BilateralHT"), + "siamese_fused": ("bilat_test", "SiameseHT"), + # "ensemble_fused_head": ("ensemble_test", "Ensemble\n+clinical+head"), + "logit_mlp_head": ("ensemble_test", "Ensemble\n+Fusion head"), +} +PHASE5_BASELINE = "ensemble_fused" + + +def analyze_phase5(out_dir: Path): + print("\n=== Phase 5 ===") + aucs = {} + for run, (key, _) in PHASE5_RUNS.items(): + a = load_rep_aucs(RESULTS_ROOT / "phase5" / run, key) + aucs[run] = a + print(f" {run:25s} AUC={np.mean(a):.3f}±{np.std(a):.3f} n={len(a)}") + + base_run = PHASE5_BASELINE + base_aucs = aucs[base_run] + others = [(r, PHASE5_RUNS[r][1]) for r in PHASE5_RUNS if r != base_run] + others.sort(key=lambda x: -np.mean(aucs[x[0]]) if len(aucs.get(x[0], [])) else float("inf")) + ordered = [(base_run, PHASE5_RUNS[base_run][1])] + others + run_keys = [r for r, _ in ordered] + labels = [lbl for _, lbl in ordered] + data_list = [aucs[r] for r in run_keys] + aucs_by_label = {lbl: aucs[r] for r, lbl in ordered} + + fig, ax = plt.subplots(figsize=(11, 5)) + fig.suptitle("Phase 5 — Full HyperTower: Bilateral + Clinical", + fontsize=FSIZE + 2, fontweight="bold") + boxplot_panel(ax, data_list, labels, base_idx=0, + base_aucs=base_aucs, all_aucs_by_label=aucs_by_label) + ax.set_ylim(0.78, None) + + fig.tight_layout() + path = out_dir / "phase5_analysis.png" + fig.savefig(path, dpi=150, bbox_inches="tight") + plt.close(fig) + print(f" Saved: {path}") + + +# ── Cross-phase summary ─────────────────────────────────────────────────────── + +def analyze_cross_phase(out_dir: Path): + """Single figure tracing the best model from each phase.""" + print("\n=== Cross-phase progression ===") + trajectory = [ + ("Phase 2\nresnet50 proper", "phase2", "imageonly_resnet50_proper", "classic_test"), + ("Phase 2\nrefugelike proper", "phase2", "imageonly_refugelike_proper", "classic_test"), + ("Phase 3\n+IOP ratio\n+drop raw", "phase3", "iop_ratio_drop_raw", "classic_test"), + ("Phase 4\nensemble\n(image only)", "phase4", "ensemble", "ensemble_test"), + ("Phase 5\nensemble\n+clinical", "phase5", "ensemble_fused", "ensemble_test"), + ("Phase 5\nensemble\n+clinical+head","phase5","ensemble_fused_head", "ensemble_test"), + ] + + labels, means, stds, all_aucs = [], [], [], [] + for lbl, phase, run, key in trajectory: + a = load_rep_aucs(RESULTS_ROOT / phase / run, key) + labels.append(lbl) + means.append(np.mean(a) if len(a) else np.nan) + stds.append(np.std(a) if len(a) else np.nan) + all_aucs.append(a) + print(f" {lbl.replace(chr(10),' '):35s} AUC={means[-1]:.3f}±{stds[-1]:.3f} n={len(a)}") + + fig, ax = plt.subplots(figsize=(10, 4)) + x = np.arange(len(labels)) + ax.errorbar(x, means, yerr=stds, fmt="o-", linewidth=2, markersize=7, + capsize=4, color="#4c72b0") + ax.set_xticks(x) + ax.set_xticklabels(labels, fontsize=8) + ax.set_ylabel("Test AUC (10-rep mean ± std)") + ax.set_title("HyperTower — Model Progression Across Phases", fontsize=12, fontweight="bold") + ax.set_ylim(0.75, 0.95) + ax.axhline(means[0], color="gray", linewidth=1, linestyle=":", alpha=0.5, label="Phase 2 baseline") + ax.legend(fontsize=8) + ax.grid(axis="y", alpha=0.3) + + fig.tight_layout() + path = out_dir / "cross_phase_progression.png" + fig.savefig(path, dpi=150, bbox_inches="tight") + plt.close(fig) + print(f" Saved: {path}") + + +# ── Main ───────────────────────────────────────────────────────────────────── + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--phase", type=int, choices=[1, 2, 3, 4, 5], + help="Run only this phase (default: all)") + ap.add_argument("--out", type=Path, + default=Path(__file__).resolve().parents[3] / "v3" / "figures", + help="Output directory for figures") + args = ap.parse_args() + + args.out.mkdir(parents=True, exist_ok=True) + + run_all = args.phase is None + if run_all or args.phase == 1: + analyze_phase1(args.out) + if run_all or args.phase == 2: + analyze_phase2(args.out) + if run_all or args.phase == 3: + analyze_phase3(args.out) + if run_all or args.phase == 4: + analyze_phase4(args.out) + if run_all or args.phase == 5: + analyze_phase5(args.out) + if run_all: + analyze_cross_phase(args.out) + + +if __name__ == "__main__": + main() diff --git a/v3/scripts/output_analysis/explainability/__init__.py b/v3/scripts/output_analysis/explainability/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/v3/scripts/output_analysis/explainability/comparison_panel_phase5.py b/v3/scripts/output_analysis/explainability/comparison_panel_phase5.py new file mode 100644 index 0000000..1a9d3d4 --- /dev/null +++ b/v3/scripts/output_analysis/explainability/comparison_panel_phase5.py @@ -0,0 +1,264 @@ +""" +Phase 5 comparison panel — ROC curves + fusion event summaries. + +Layout: + Top row (1 × 3) — ROC curves: Single HyperTower | Bilateral Ensemble | Fused Head + Bottom rows (2 × 1) — Fusion event summary (full-width) for Ensemble then Fused Head + (Single mode has fused-only bridge; no meaningful fusion events) + +All data derived from predictions_test.csv — no checkpoints required. + +Usage: + python -m v3.scripts.output_analysis.explainability.comparison_panel_phase5 +""" +from __future__ import annotations + +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.patches as mpatches +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from sklearn.metrics import roc_auc_score, roc_curve + +REPO_ROOT = Path(__file__).resolve().parents[4] +RESULTS_ROOT = REPO_ROOT / "v3" / "results" +FIGURES_ROOT = REPO_ROOT / "v3" / "figures" / "explainability" +CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData" + +RUNS = [ + {"label": "Single HyperTower", "run": "phase5/single_fused", + "tower_path": "binary/single", "is_single": False}, + {"label": "Bilateral Ensemble", "run": "phase5/ensemble_fused", + "tower_path": "binary/ensemble", "is_single": False}, + {"label": "Fused Head", "run": "phase5/logit_mlp_head", + "tower_path": "binary/ensemble", "is_single": False}, +] + +# Event taxonomy (img=image tower, md=clinical tower) +_EVENT_KEYS = [ + "full_correction", "img_assist", "md_assist", + "full_error", "img_drag", "md_drag", + "concordant_correct", "concordant_wrong", +] +_EVENT_COLORS = [ + "#2ca02c", "#98df8a", "#b5cf6b", # positive + "#d62728", "#ff9896", "#ffbb78", # negative + "#aec7e8", "#c5b0d5", # concordant +] +_POSITIVE_KEYS = _EVENT_KEYS[:3] +_NEGATIVE_KEYS = _EVENT_KEYS[3:6] +_DISAGREE_KEYS = _POSITIVE_KEYS + _NEGATIVE_KEYS # exclude concordant + + +# ── Data loading ────────────────────────────────────────────────────────────── + +def load_pooled(run: str, tower_path: str) -> pd.DataFrame: + run_dir = RESULTS_ROOT / run + rows = [] + for rep in sorted(run_dir.glob("rep*")): + tm = rep / tower_path + if not tm.exists(): + continue + for fold in sorted(tm.glob("fold[0-9]")): + csv = fold / "predictions_test.csv" + if csv.exists(): + df = pd.read_csv(csv) + df["rep"] = rep.name + df["fold"] = fold.name + rows.append(df) + if not rows: + raise FileNotFoundError(f"No predictions found under {run_dir}/{tower_path}") + return pd.concat(rows, ignore_index=True) + + +def classify_events(df: pd.DataFrame) -> pd.DataFrame: + """Add event_type column based on pred_fused/pred_img/pred_md vs y_true.""" + df = df.copy() + y = df["y_true"].values + pf = df["pred_fused"].values + pi = df["pred_img"].values + pm = df["pred_md"].values + + fused_ok = pf == y + img_ok = pi == y + md_ok = pm == y + + def _classify(fo, io, mo): + if fo and io and mo: return "concordant_correct" + if not fo and not io and not mo: return "concordant_wrong" + if fo and not io and not mo: return "full_correction" + if fo and io and not mo: return "img_assist" + if fo and not io and mo: return "md_assist" + if not fo and io and mo: return "full_error" + if not fo and not io and mo: return "img_drag" + if not fo and io and not mo: return "md_drag" + return "other" + + df["event_type"] = [_classify(fo, io, mo) + for fo, io, mo in zip(fused_ok, img_ok, md_ok)] + # conf_delta: fused prob minus average of img/md + df["conf_fused"] = df["prob_fused_c1"] + df["conf_img"] = df["prob_img_c1"] + df["conf_md"] = df["prob_md_c1"] + df["conf_delta"] = df["conf_fused"] - 0.5 * (df["conf_img"] + df["conf_md"]) + return df + + +# ── ROC panel ───────────────────────────────────────────────────────────────── + +def _draw_roc(ax, df: pd.DataFrame, label: str, color: str) -> None: + """Draw per-fold ROC curves (faint) + mean ROC (bold) on ax.""" + fold_aucs = [] + for (rep, fold), grp in df.groupby(["rep", "fold"]): + if grp["y_true"].nunique() < 2: + continue + fpr, tpr, _ = roc_curve(grp["y_true"], grp["prob_fused_c1"]) + ax.plot(fpr, tpr, color=color, alpha=0.12, lw=0.8) + fold_aucs.append(roc_auc_score(grp["y_true"], grp["prob_fused_c1"])) + + # Mean ROC via interpolation + mean_fpr = np.linspace(0, 1, 200) + tprs = [] + for (rep, fold), grp in df.groupby(["rep", "fold"]): + if grp["y_true"].nunique() < 2: + continue + fpr, tpr, _ = roc_curve(grp["y_true"], grp["prob_fused_c1"]) + tprs.append(np.interp(mean_fpr, fpr, tpr)) + mean_tpr = np.mean(tprs, axis=0) + mean_auc = np.mean(fold_aucs) + std_auc = np.std(fold_aucs) + ax.plot(mean_fpr, mean_tpr, color=color, lw=2.2, + label=f"Mean AUC = {mean_auc:.3f} ± {std_auc:.3f}") + ax.fill_between(mean_fpr, + np.percentile(tprs, 25, axis=0), + np.percentile(tprs, 75, axis=0), + color=color, alpha=0.12) + ax.plot([0, 1], [0, 1], "k--", lw=0.7, alpha=0.5) + ax.set_xlim(-0.02, 1.02); ax.set_ylim(-0.02, 1.02) + ax.set_xlabel("False Positive Rate", fontsize=9) + ax.set_ylabel("True Positive Rate", fontsize=9) + ax.set_title(label, fontsize=10, fontweight="bold") + ax.legend(fontsize=8, loc="lower right") + ax.grid(alpha=0.25) + + +# ── Fusion summary panel ────────────────────────────────────────────────────── + +def _draw_fusion_summary(axes_row, df: pd.DataFrame, label: str) -> None: + """Draw 3-panel fusion summary (disagreement events only) on axes_row (list of 3 axes).""" + event_color = dict(zip(_EVENT_KEYS, _EVENT_COLORS)) + event_labels = { + "full_correction": "Full correction\n(both wrong → right)", + "img_assist": "Img assist\n(img✓ md✗ → right)", + "md_assist": "MD assist\n(md✓ img✗ → right)", + "full_error": "Full error\n(both right → wrong)", + "img_drag": "Img drag\n(img✗ md✓ → wrong)", + "md_drag": "MD drag\n(md✗ img✓ → wrong)", + } + + # Only count disagreement events (exclude concordant) + counts = {k: (df["event_type"] == k).sum() for k in _DISAGREE_KEYS} + + # Panel 0: totals bar (positive vs negative) + ax = axes_row[0] + for bar_x, keys in ((0, _POSITIVE_KEYS), (1, _NEGATIVE_KEYS)): + bot = 0 + for k in keys: + c = int(counts[k]) + ax.bar(bar_x, c, bottom=bot, color=event_color[k], width=0.5) + if c > 0: + ax.text(bar_x, bot + c / 2, str(c), ha="center", va="center", + fontsize=8, fontweight="bold") + bot += c + ax.set_xticks([0, 1]); ax.set_xticklabels(["Positive\nevents", "Negative\nevents"]) + ax.set_ylabel("Count (all folds)") + patches = [mpatches.Patch(color=event_color[k], label=event_labels[k].split("\n")[0]) + for k in _DISAGREE_KEYS if counts[k] > 0] + ax.legend(handles=patches, fontsize=6, loc="upper right") + ax.set_title(f"{label}\nDisagreement event totals", fontsize=9) + + # Panel 1: per-fold stacked bar (disagreement events only) + ax = axes_row[1] + fold_groups = sorted(df.groupby(["rep", "fold"]), key=lambda x: x[0]) + x = np.arange(len(fold_groups)) + pos_bot = np.zeros(len(fold_groups)) + neg_bot = np.zeros(len(fold_groups)) + for k, color in zip(_POSITIVE_KEYS, _EVENT_COLORS[:3]): + vals = np.array([(g["event_type"] == k).sum() for _, g in fold_groups], dtype=float) + ax.bar(x, vals, bottom=pos_bot, color=color, width=0.6) + pos_bot += vals + for k, color in zip(_NEGATIVE_KEYS, _EVENT_COLORS[3:6]): + vals = np.array([(g["event_type"] == k).sum() for _, g in fold_groups], dtype=float) + ax.bar(x + 0.65, vals, bottom=neg_bot, color=color, width=0.6) + neg_bot += vals + ax.set_xticks([]) + ax.set_xlabel("Fold", fontsize=8) + ax.set_ylabel("Count"); ax.set_title("Per-fold breakdown\n(left=positive, right=negative)", fontsize=9) + + # Panel 2: img vs md confidence scatter (disagreement events only) + ax = axes_row[2] + for k in _DISAGREE_KEYS: + sub = df[df["event_type"] == k] + if len(sub) == 0: + continue + ax.scatter(sub["conf_img"], sub["conf_md"], c=event_color[k], + alpha=0.65, s=30, edgecolors="none", + label=event_labels[k].split("\n")[0]) + ax.plot([0, 1], [0, 1], "k--", lw=0.5, alpha=0.4) + ax.set_xlabel("P(Glaucoma) — Image head"); ax.set_ylabel("P(Glaucoma) — MD head") + ax.legend(fontsize=5.5, loc="lower right") + ax.set_title("Tower confidence space\n(disagreement events only)", fontsize=9) + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main(): + ROC_COLORS = ["#4c72b0", "#dd8452", "#55a868"] + + print("Loading predictions ...") + datasets = [] + for cfg, color in zip(RUNS, ROC_COLORS): + df = load_pooled(cfg["run"], cfg["tower_path"]) + df = classify_events(df) + datasets.append((cfg, df, color)) + + fusion_runs = [(cfg, df, color) for cfg, df, color in datasets] + + # ── Layout ──────────────────────────────────────────────────────────────── + # Row 0: 3 ROC axes + # Rows 1,2: 2 fusion summary strips (5 axes each, spanning full width) + n_fusion = len(fusion_runs) + fig = plt.figure(figsize=(20, 6 + 4.5 * n_fusion)) + gs = fig.add_gridspec( + 1 + n_fusion, 1, + height_ratios=[5] + [4.5] * n_fusion, + hspace=0.35, + ) + + # ROC row — subdivide into 3 + roc_gs = gs[0].subgridspec(1, 3, wspace=0.28) + for i, (cfg, df, color) in enumerate(datasets): + ax = fig.add_subplot(roc_gs[i]) + _draw_roc(ax, df, cfg["label"], color) + + # Fusion rows (3 panels each) + for fi, (cfg, df, color) in enumerate(fusion_runs): + fus_gs = gs[1 + fi].subgridspec(1, 3, wspace=0.32) + axes_row = [fig.add_subplot(fus_gs[j]) for j in range(3)] + _draw_fusion_summary(axes_row, df, cfg["label"]) + + fig.suptitle("Phase 5 — Model Comparison: ROC Curves & Fusion Event Analysis", + fontsize=13, fontweight="bold", y=1.01) + + out = FIGURES_ROOT / "comparison_panel_phase5.png" + out.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(out, dpi=150, bbox_inches="tight") + plt.close(fig) + print(f"Saved: {out}") + + +if __name__ == "__main__": + main() diff --git a/v3/scripts/output_analysis/explainability/confidence_strips.py b/v3/scripts/output_analysis/explainability/confidence_strips.py new file mode 100644 index 0000000..ae00733 --- /dev/null +++ b/v3/scripts/output_analysis/explainability/confidence_strips.py @@ -0,0 +1,615 @@ +""" +Phase 5 explainability — confidence strips and head comparison. + +Works from saved prediction CSVs. If patient_id column is present (requires +a re-run after the v3_hypertower.py update), points are colored by VFI +severity group. Otherwise falls back to a single color per true class. + +VFI severity groups (VF_MD from clinical data): + Early VF_MD > -6 + Moderate VF_MD -6 to -12 + Severe VF_MD < -12 + +Produces (all in figures/explainability/): + confidence_strips.png — vertical strip: P(glaucoma) by true class, VFI colored + head_comparison.png — fused vs img vs md distributions side by side + +Usage: + python -m v3.scripts.output_analysis.explainability.confidence_strips + python -m v3.scripts.output_analysis.explainability.confidence_strips \ + --run phase5/logit_mlp_head --clinical-dir Papila/ClinicalData +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import matplotlib.patches as mpatches +import numpy as np +import pandas as pd + +REPO_ROOT = Path(__file__).resolve().parents[4] +RESULTS_ROOT = REPO_ROOT / "v3" / "results" +FIGURES_ROOT = REPO_ROOT / "v3" / "figures" +CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData" + +FONT = "DejaVu Sans" + +# Colour palette +C_NORMAL = "#78909C" # blue-grey — healthy controls (no VFI staging) +C_EARLY = "#29B6F6" # sky-blue — glaucoma, early VFI loss +C_MODERATE = "#FFB300" # amber — glaucoma, moderate VFI loss +C_SEVERE = "#E53935" # vivid red — glaucoma, severe VFI loss +C_UNKNOWN = "#BDBDBD" # light grey — glaucoma, VFI not recorded + +SEV_LABELS = { + "normal": "Normal", + "early": "Glaucoma — early (VF_MD > −6)", + "moderate": "Glaucoma — moderate (−12 to −6)", + "severe": "Glaucoma — severe (VF_MD < −12)", + "unknown": "Glaucoma — VF_MD not recorded", +} +SEV_COLORS = { + "normal": C_NORMAL, + "early": C_EARLY, + "moderate": C_MODERATE, + "severe": C_SEVERE, + "unknown": C_UNKNOWN, +} + +HEAD_COLORS = {"fused": "#d4a017", "img": "#4e8d3a", "md": "#4c72b0"} +HEAD_LABELS = { + "fused": "Fused head", + "img": "Image-only head", + "md": "Clinical-only head", +} + + +# ── VFI data ────────────────────────────────────────────────────────────────── + + +def load_vfi(clinical_dir: Path) -> pd.DataFrame: + """Return DataFrame with columns [patient_id (int), vf_md (float), severity (str)]. + Only includes patients in the binary study (Diagnosis 0=Normal, 1=Glaucoma). + """ + od = pd.read_excel(clinical_dir / "patient_data_od.xlsx", header=1) + os_ = pd.read_excel(clinical_dir / "patient_data_os.xlsx", header=1) + + def _clean(df, eye): + df = df.copy() + if "Patient ID" not in df.columns and "ID" in df.columns: + df.rename(columns={"ID": "Patient ID"}, inplace=True) + df["Patient ID"] = ( + df["Patient ID"].astype(str).str.extract(r"(\d+)")[0].astype(int) + ) + df["Diagnosis"] = pd.to_numeric(df["Diagnosis"], errors="coerce") + df["VF_MD"] = pd.to_numeric(df["VF_MD"], errors="coerce") + # PAPILA: 0=Normal, 1=Glaucoma, 2=Suspect — keep only binary patients + df = df[df["Diagnosis"].isin([0, 1])].copy() + df["eye"] = eye + return df[["Patient ID", "Diagnosis", "VF_MD", "eye"]] + + combined = pd.concat([_clean(od, "OD"), _clean(os_, "OS")], ignore_index=True) + + # Per patient: modal diagnosis, worst (most negative) VF_MD across eyes + diag = ( + combined.groupby("Patient ID")["Diagnosis"] + .agg(lambda x: x.mode().iloc[0]) + .reset_index() + ) + vf = combined.groupby("Patient ID")["VF_MD"].min().reset_index() + worst = diag.merge(vf, on="Patient ID").rename( + columns={"Patient ID": "patient_id", "VF_MD": "vf_md", "Diagnosis": "diagnosis"} + ) + + def _severity(row): + if int(row["diagnosis"]) == 0: + return "normal" # healthy control — no VFI staging + v = row["vf_md"] + if pd.isna(v): + return "unknown" # glaucoma, no VFI recorded + if v > -6: + return "early" + if v > -12: + return "moderate" + return "severe" + + worst["severity"] = worst.apply(_severity, axis=1) + return worst + + +# ── Prediction loading ──────────────────────────────────────────────────────── + + +def load_all_predictions( + run_dir: Path, tower_path: str = "binary/ensemble" +) -> pd.DataFrame: + """Pool predictions_test.csv across all reps and folds.""" + rows = [] + for rep_dir in sorted(run_dir.glob("rep*")): + tm_dir = rep_dir / tower_path + if not tm_dir.exists(): + continue + for fold_dir in sorted(tm_dir.glob("fold[0-9]")): + csv_path = fold_dir / "predictions_test.csv" + if not csv_path.exists(): + continue + df = pd.read_csv(csv_path) + df["rep"] = rep_dir.name + df["fold"] = fold_dir.name + rows.append(df) + if not rows: + raise FileNotFoundError(f"No predictions_test.csv found under {run_dir}") + return pd.concat(rows, ignore_index=True) + + +# ── Figure 1: Confidence strips (vertical) ─────────────────────────────────── + + +def make_confidence_strips( + df: pd.DataFrame, vfi: pd.DataFrame | None, out_path: Path +) -> None: + """ + Vertical strip plot: x = true class, y = P(glaucoma). + Points colored by VFI severity if patient_id column available, else uniform. + """ + rng = np.random.default_rng(42) + has_vfi = ( + vfi is not None + and "patient_id" in df.columns + and df["patient_id"].notna().any() + ) + + if has_vfi: + df = df.copy() + df["patient_id"] = pd.to_numeric(df["patient_id"], errors="coerce").astype( + "Int64" + ) + vfi_merge = vfi.copy() + vfi_merge["patient_id"] = vfi_merge["patient_id"].astype("Int64") + df = df.merge( + vfi_merge[["patient_id", "severity"]], on="patient_id", how="left" + ) + df["severity"] = df["severity"].fillna("unknown") + else: + df["severity"] = "unknown" + + fig, ax = plt.subplots(figsize=(6, 7)) + fig.patch.set_facecolor("#e8e8e8") + ax.set_facecolor("#e8e8e8") + + x_pos = {0: 0.0, 1: 1.0} + jitter_scale = 0.18 + + # Draw in severity order so severe is on top + sev_order = ["normal", "unknown", "early", "moderate", "severe"] + sev_alpha = { + "normal": 0.40, + "early": 0.55, + "moderate": 0.70, + "severe": 0.85, + "unknown": 0.35, + } + sev_size = {"normal": 6, "early": 8, "moderate": 10, "severe": 12, "unknown": 6} + + for sev in sev_order: + mask = df["severity"] == sev + if not mask.any(): + continue + sub = df[mask] + jitter = rng.uniform(-jitter_scale, jitter_scale, len(sub)) + x = np.array([x_pos[int(v)] for v in sub["y_true"]]) + jitter + ax.scatter( + x, + sub["prob_fused_c1"].values, + c=SEV_COLORS[sev], + s=sev_size[sev], + alpha=sev_alpha[sev], + linewidths=0, + zorder=3, + label=SEV_LABELS[sev], + ) + + # Median lines per class + for cls, xc in x_pos.items(): + med = np.median(df.loc[df["y_true"] == cls, "prob_fused_c1"]) + ax.plot( + [xc - jitter_scale - 0.04, xc + jitter_scale + 0.04], + [med, med], + color="#222", + lw=2.0, + zorder=5, + ) + + ax.axhline(0.5, color="#888", lw=1.2, ls="--", alpha=0.7, zorder=2) + ax.set_xticks([0, 1]) + ax.set_xticklabels(["Normal", "Glaucoma"], fontsize=11) + ax.set_ylabel("Predicted P(Glaucoma)", fontsize=11) + ax.set_ylim(-0.04, 1.04) + ax.set_xlim(-0.55, 1.55) + ax.set_title( + "Confidence Strips — Fused Head\n(Phase 5, all folds)", + fontsize=12, + fontweight="bold", + ) + ax.grid(axis="y", alpha=0.3, zorder=1) + + # Legend — only show groups that appear + handles, labels = ax.get_legend_handles_labels() + if handles: + ax.legend( + handles=handles, + labels=labels, + fontsize=8.5, + loc="upper center", + framealpha=0.75, + ncol=2, + ) + + if not has_vfi: + ax.text( + 0.98, + 0.02, + "Re-run with updated v3_hypertower.py\nto enable VFI severity coloring", + transform=ax.transAxes, + ha="right", + va="bottom", + fontsize=7.5, + color="#888", + style="italic", + ) + + fig.tight_layout() + out_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(out_path, dpi=180, bbox_inches="tight") + plt.close(fig) + print(f" Saved: {out_path}") + + +# ── Figure 2: Head comparison ───────────────────────────────────────────────── + + +def make_head_comparison(df: pd.DataFrame, out_path: Path) -> None: + """Side-by-side violin + strip of P(glaucoma) by true class for each head.""" + heads = ["fused", "img", "md"] + prob_cols = {"fused": "prob_fused_c1", "img": "prob_img_c1", "md": "prob_md_c1"} + + rng = np.random.default_rng(42) + fig, axes = plt.subplots(1, 3, figsize=(11, 4.5), sharey=True) + fig.patch.set_facecolor("#e8e8e8") + fig.suptitle( + "Head Comparison — P(Glaucoma) by True Class (Phase 5, all folds)", + fontsize=12, + fontweight="bold", + ) + + c_normal = "#4c72b0" + c_glaucoma = "#c44e52" + + for ax, head in zip(axes, heads): + ax.set_facecolor("#e8e8e8") + col = prob_cols[head] + data_by_class = [df.loc[df["y_true"] == cls, col].values for cls in [0, 1]] + + vp = ax.violinplot( + data_by_class, + positions=[0, 1], + widths=0.6, + showmedians=True, + showextrema=False, + ) + for body, color in zip(vp["bodies"], [c_normal, c_glaucoma]): + body.set_facecolor(color) + body.set_alpha(0.35) + vp["cmedians"].set_color("#222") + vp["cmedians"].set_linewidth(2) + + for cls, color in zip([0, 1], [c_normal, c_glaucoma]): + vals = data_by_class[cls] + jitter = rng.uniform(-0.12, 0.12, len(vals)) + ax.scatter( + cls + jitter, vals, color=color, s=4, alpha=0.30, linewidths=0, zorder=3 + ) + + ax.axhline(0.5, color="#888", lw=1.0, ls="--", alpha=0.6) + ax.set_xticks([0, 1]) + ax.set_xticklabels(["Normal", "Glaucoma"], fontsize=9) + ax.set_title( + HEAD_LABELS[head], fontsize=10, fontweight="bold", color=HEAD_COLORS[head] + ) + ax.set_ylim(-0.05, 1.05) + ax.grid(axis="y", alpha=0.3) + if head == "fused": + ax.set_ylabel("Predicted P(Glaucoma)", fontsize=10) + + from sklearn.metrics import roc_auc_score + + try: + auc = roc_auc_score(df["y_true"], df[col]) + ax.text( + 0.97, + 0.04, + f"AUC = {auc:.3f}", + transform=ax.transAxes, + ha="right", + va="bottom", + fontsize=9, + color="#333", + bbox=dict(facecolor="white", alpha=0.65, edgecolor="none", pad=2), + ) + except Exception: + pass + + fig.tight_layout() + out_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(out_path, dpi=180, bbox_inches="tight") + plt.close(fig) + print(f" Saved: {out_path}") + + +# ── Multi-model comparison strip ────────────────────────────────────────────── + + +def make_comparison_strips( + run_configs: list[dict], vfi: pd.DataFrame, out_path: Path +) -> None: + """ + Side-by-side confidence strips for multiple runs. + Each config: {"label": str, "df": DataFrame}. + """ + from sklearn.metrics import roc_auc_score + + rng = np.random.default_rng(42) + + n = len(run_configs) + fig, axes = plt.subplots(1, n, figsize=(4.5 * n, 7), sharey=True) + if n == 1: + axes = [axes] + fig.patch.set_facecolor("#e8e8e8") + fig.suptitle( + "Confidence Strips by Model (Phase 5, all folds)", + fontsize=13, + fontweight="bold", + ) + + sev_order = ["normal", "unknown", "early", "moderate", "severe"] + sev_alpha = { + "normal": 0.40, + "early": 0.55, + "moderate": 0.70, + "severe": 0.85, + "unknown": 0.35, + } + sev_size = {"normal": 6, "early": 8, "moderate": 10, "severe": 12, "unknown": 6} + x_pos = {0: 0.0, 1: 1.0} + jitter_scale = 0.18 + + for ax, cfg in zip(axes, run_configs): + ax.set_facecolor("#e8e8e8") + df = cfg["df"] + + # Attach VFI severity + df = df.copy() + df["patient_id"] = pd.to_numeric(df["patient_id"], errors="coerce").astype( + "Int64" + ) + vfi_m = vfi.copy() + vfi_m["patient_id"] = vfi_m["patient_id"].astype("Int64") + df = df.merge(vfi_m[["patient_id", "severity"]], on="patient_id", how="left") + df["severity"] = df["severity"].fillna("unknown") + + for sev in sev_order: + mask = df["severity"] == sev + if not mask.any(): + continue + sub = df[mask] + jitter = rng.uniform(-jitter_scale, jitter_scale, len(sub)) + x = np.array([x_pos[int(v)] for v in sub["y_true"]]) + jitter + ax.scatter( + x, + sub["prob_fused_c1"].values, + c=SEV_COLORS[sev], + s=sev_size[sev], + alpha=sev_alpha[sev], + linewidths=0, + zorder=3, + ) + + # IQR box + median line per class + iqr_w = 0.06 + xticklabels = [] + for cls, xc in x_pos.items(): + vals = df.loc[df["y_true"] == cls, "prob_fused_c1"] + med = np.median(vals) + q25 = np.percentile(vals, 25) + q75 = np.percentile(vals, 75) + # Subtle translucent IQR box + ax.add_patch( + plt.Rectangle( + (xc - iqr_w, q25), + 2 * iqr_w, + q75 - q25, + facecolor="#555", + alpha=0.18, + linewidth=0, + zorder=4, + ) + ) + # Median line + ax.plot( + [xc - jitter_scale - 0.04, xc + jitter_scale + 0.04], + [med, med], + color="#222", + lw=2.0, + zorder=5, + label="Median" if cls == 0 else None, + ) + # TP / TN rate below x-label + if cls == 1: + rate = (vals > 0.5).mean() * 100 + xticklabels.append(f"Glaucoma\nTP {rate:.0f}%") + else: + rate = (vals <= 0.5).mean() * 100 + xticklabels.append(f"Normal\nTN {rate:.0f}%") + + ax.axhline(0.5, color="#888", lw=1.2, ls="--", alpha=0.7, zorder=2) + ax.set_xticks([0, 1]) + ax.set_xticklabels(xticklabels, fontsize=10) + ax.set_title(cfg["label"], fontsize=11, fontweight="bold") + ax.set_ylim(-0.04, 1.04) + ax.set_xlim(-0.55, 1.55) + ax.grid(axis="y", alpha=0.3, zorder=1) + + try: + fold_aucs = [ + roc_auc_score(g["y_true"], g["prob_fused_c1"]) + for _, g in df.groupby(["rep", "fold"]) + if g["y_true"].nunique() > 1 + ] + mean_auc = np.mean(fold_aucs) + std_auc = np.std(fold_aucs) + ax.text( + 0.65, + 0.0, + f"AUC = {mean_auc:.3f} ± {std_auc:.3f}", + transform=ax.transAxes, + ha="right", + va="bottom", + fontsize=9, + color="#333", + bbox=dict(facecolor="white", alpha=0.65, edgecolor="none", pad=2), + ) + except Exception: + pass + + axes[0].set_ylabel("Predicted P(Glaucoma)", fontsize=11) + + # Shared legend + legend_patches = [ + mpatches.Patch(color=SEV_COLORS[s], label=SEV_LABELS[s]) + for s in ["normal", "early", "moderate", "severe", "unknown"] + ] + fig.legend( + handles=legend_patches, + fontsize=9, + loc="lower center", + ncol=len(legend_patches), + framealpha=0.75, + bbox_to_anchor=(0.5, -0.02), + ) + + fig.tight_layout(rect=[0, 0.06, 1, 1]) + out_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(out_path, dpi=180, bbox_inches="tight") + plt.close(fig) + print(f" Saved: {out_path}") + + +# ── Main ────────────────────────────────────────────────────────────────────── + +# Default runs shown in the comparison +DEFAULT_RUNS = [ + { + "run": "phase4/single", + "tower_path": "binary/single", + "label": "Single HyperTower", + }, + { + "run": "phase5/ensemble_fused", + "tower_path": "binary/ensemble", + "label": "Bilateral Ensemble", + }, + { + "run": "phase5/logit_mlp_head", + "tower_path": "binary/ensemble", + "label": "Fused Head", + }, +] + + +def _aggregate_eye_to_patient(df: pd.DataFrame) -> pd.DataFrame: + """ + Single-mode predictions are eye-level (2 rows per patient per fold). + In the test loader, OD rows come first (sorted patient ID order) then OS. + Average the two eyes to get one patient-level row per fold. + """ + rows = [] + prob_cols = [c for c in df.columns if c.startswith("prob_")] + pred_cols = [c for c in df.columns if c.startswith("pred_")] + + for (rep, fold), grp in df.groupby(["rep", "fold"]): + n = len(grp) + half = n // 2 + od = grp.iloc[:half].reset_index(drop=True) + os_ = grp.iloc[half:].reset_index(drop=True) + pat = od.copy() + for col in prob_cols: + pat[col] = (od[col].values + os_[col].values) / 2 + for col in pred_cols: + pat[col] = (pat[col.replace("pred_", "prob_") + "_c1"] >= 0.5).astype(int) + rows.append(pat) + + return pd.concat(rows, ignore_index=True) + + +def _load_run(run: str, tower_path: str, clinical_dir: Path) -> pd.DataFrame: + run_dir = RESULTS_ROOT / run + print(f" Loading {run} ...") + df = load_all_predictions(run_dir, tower_path=tower_path) + if tower_path.endswith("/single"): + from v3.scripts.output_analysis.explainability.fold_patient_ids import ( + attach_patient_ids_single, + ) + + df = attach_patient_ids_single(df, clinical_dir=clinical_dir, batch_size=8) + else: + from v3.scripts.output_analysis.explainability.fold_patient_ids import ( + attach_patient_ids, + ) + + df = attach_patient_ids(df, clinical_dir=clinical_dir) + return df + + +def main(): + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument( + "--run", default="phase5/logit_mlp_head", help="Single run for standalone plots" + ) + ap.add_argument("--tower-path", default="binary/ensemble") + ap.add_argument("--clinical-dir", type=Path, default=CLINICAL_DIR) + ap.add_argument("--out", type=Path, default=FIGURES_ROOT / "explainability") + args = ap.parse_args() + + print("Loading VFI data ...") + vfi = load_vfi(args.clinical_dir) + + # ── Single-run plots (strips + head comparison) ────────────────────────── + df = _load_run(args.run, args.tower_path, args.clinical_dir) + make_confidence_strips(df, vfi, args.out / "confidence_strips.png") + make_head_comparison(df, args.out / "head_comparison.png") + + # ── Multi-model comparison ─────────────────────────────────────────────── + print("Building comparison strips ...") + run_configs = [] + for cfg in DEFAULT_RUNS: + try: + df_r = _load_run(cfg["run"], cfg["tower_path"], args.clinical_dir) + run_configs.append({"label": cfg["label"], "df": df_r}) + except FileNotFoundError as e: + print(f" Skipping {cfg['run']}: {e}") + if run_configs: + make_comparison_strips( + run_configs, vfi, args.out / "confidence_strips_comparison.png" + ) + + +if __name__ == "__main__": + main() diff --git a/v3/scripts/output_analysis/explainability/fold_patient_ids.py b/v3/scripts/output_analysis/explainability/fold_patient_ids.py new file mode 100644 index 0000000..74eab23 --- /dev/null +++ b/v3/scripts/output_analysis/explainability/fold_patient_ids.py @@ -0,0 +1,181 @@ +""" +Derives test-set patient IDs for any (rep, fold) without re-running training. + +The split is fully deterministic: fold_seed = rep_seed_start + rep * rep_seed_step. +build_samples() groups by patient_id with sort=True (pandas default), and the +test DataLoader uses shuffle=False — so rows in predictions_test.csv are always +in ascending Patient ID order within each test fold. + +Usage: + from v3.scripts.output_analysis.explainability.fold_patient_ids import get_test_patient_ids + pids = get_test_patient_ids(rep=0, fold=2) # list of int patient IDs, sorted + + # Attach to a pooled predictions DataFrame: + df = attach_patient_ids(df, clinical_dir=...) +""" +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +REPO_ROOT = Path(__file__).resolve().parents[4] +CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData" + +# These match the defaults in run_cv.py +_REP_SEED_START = 100 +_REP_SEED_STEP = 100 +_N_SPLITS = 5 +_EVAL_MODE = "binary" +_LABEL_COL = "Diagnosis" +_PATIENT_COL = "Patient ID" + + +@lru_cache(maxsize=4) +def _load_clinical(clinical_dir: Path) -> pd.DataFrame: + """ + Load OD/OS Excel sheets, extract Patient ID + Diagnosis, binary-filter. + Returns a DataFrame with one row per eye (OD+OS stacked), columns: + [Patient ID, Diagnosis, eyeID, VF_MD]. + """ + od = pd.read_excel(clinical_dir / "patient_data_od.xlsx", header=1) + os_ = pd.read_excel(clinical_dir / "patient_data_os.xlsx", header=1) + od["eyeID"] = "OD" + os_["eyeID"] = "OS" + df = pd.concat([od, os_], ignore_index=True) + # Raw column is "ID" (e.g. "#002"); canonicalize to "Patient ID" + if "Patient ID" not in df.columns and "ID" in df.columns: + df.rename(columns={"ID": "Patient ID"}, inplace=True) + df["Patient ID"] = df["Patient ID"].astype(str).str.extract(r"(\d+)")[0].astype(int) + df["Diagnosis"] = pd.to_numeric(df["Diagnosis"], errors="coerce") + df["VF_MD"] = pd.to_numeric(df["VF_MD"], errors="coerce") + # PAPILA encoding: 0=Normal, 1=Glaucoma, 2=Suspect + # Binary mode keeps 0 and 1, excludes Suspect (2) + df = df[df["Diagnosis"].isin([0, 1])].copy() + return df.reset_index(drop=True) + + +def _build_splits(clinical_dir: Path, fold_seed: int) -> list[Any]: + """Return list of PatientSplit for a given fold seed.""" + import sys + sys.path.insert(0, str(REPO_ROOT)) + from v3.classes.split_manager import PatientFirstSplitManager, build_patient_split_plans + + df = _load_clinical(clinical_dir) + + # Patient-level label table (mode label per patient) + patient_table = ( + df.groupby(_PATIENT_COL)[_LABEL_COL] + .agg(lambda x: x.mode().iloc[0]) + .reset_index() + ) + plans_raw = build_patient_split_plans( + patient_ids=patient_table[_PATIENT_COL].to_numpy(), + patient_labels=patient_table[_LABEL_COL].to_numpy(), + n_splits=_N_SPLITS, + seed=fold_seed, + ) + + # Wrap into PatientSplit-like objects with .test DataFrame + class _Split: + def __init__(self, test_ids): + self.test = df[df[_PATIENT_COL].isin(test_ids)].reset_index(drop=True) + + return [_Split(p.test_patient_ids) for p in plans_raw] + + +def get_test_patient_ids(rep: int, fold: int, + clinical_dir: Path = CLINICAL_DIR, + rep_seed_start: int = _REP_SEED_START, + rep_seed_step: int = _REP_SEED_STEP) -> list[int]: + """ + Return sorted list of Patient IDs in the test set for (rep, fold). + Matches the row order of predictions_test.csv for that fold. + """ + fold_seed = rep_seed_start + rep * rep_seed_step + plans = _build_splits(clinical_dir, fold_seed) + test_df = plans[fold].test + # groupby sorts by default → same order as build_samples / test loader + return sorted(test_df[_PATIENT_COL].unique().tolist()) + + +def _row_to_patient_pos(row_idx: int, n_patients: int, batch_size: int) -> int: + """ + Map a single-mode row index to its patient position in the sorted patient list. + + collect_probs_single_components (aggregate_patient=False) emits predictions + in batch-interleaved order: for each batch of B patients, OD rows come first + then OS rows. The last batch may be smaller than batch_size. + + Batch i (B patients): rows [i*2B .. i*2B+B-1] = OD + [i*2B+B .. i*2B+2B-1] = OS + Patient position = i*B + (row_in_batch % B) + """ + full = n_patients // batch_size + last_b = n_patients % batch_size + for bi in range(full): + s = bi * 2 * batch_size + if s <= row_idx < s + 2 * batch_size: + return bi * batch_size + (row_idx - s) % batch_size + if last_b > 0: + s = full * 2 * batch_size + return full * batch_size + (row_idx - s) % last_b + raise IndexError(f"row_idx {row_idx} out of range for n_patients={n_patients}") + + +def attach_patient_ids(df: pd.DataFrame, + clinical_dir: Path = CLINICAL_DIR, + rep_seed_start: int = _REP_SEED_START, + rep_seed_step: int = _REP_SEED_STEP) -> pd.DataFrame: + """ + Add a 'patient_id' column to a pooled predictions DataFrame. + Requires 'rep' and 'fold' columns (added by load_all_predictions). + The 'idx' column is the row index within each fold's test set. + For patient-level modes (ensemble): idx == patient position directly. + """ + df = df.copy() + pid_col = [] + + for _, row in df.iterrows(): + rep_idx = int(row["rep"].replace("rep", "")) + fold_idx = int(row["fold"].replace("fold", "")) + idx = int(row["idx"]) + pids = get_test_patient_ids(rep_idx, fold_idx, + clinical_dir=clinical_dir, + rep_seed_start=rep_seed_start, + rep_seed_step=rep_seed_step) + pid_col.append(pids[idx] if idx < len(pids) else None) + + df["patient_id"] = pid_col + return df + + +def attach_patient_ids_single(df: pd.DataFrame, + clinical_dir: Path = CLINICAL_DIR, + batch_size: int = 8, + rep_seed_start: int = _REP_SEED_START, + rep_seed_step: int = _REP_SEED_STEP) -> pd.DataFrame: + """ + Like attach_patient_ids but for single (eye-level) mode. + Single mode emits predictions in batch-interleaved order (see _row_to_patient_pos). + batch_size must match the --batch-size used during training (default 8). + """ + df = df.copy() + pid_col = [] + + for _, row in df.iterrows(): + rep_idx = int(row["rep"].replace("rep", "")) + fold_idx = int(row["fold"].replace("fold", "")) + idx = int(row["idx"]) + pids = get_test_patient_ids(rep_idx, fold_idx, + clinical_dir=clinical_dir, + rep_seed_start=rep_seed_start, + rep_seed_step=rep_seed_step) + patient_pos = _row_to_patient_pos(idx, len(pids), batch_size) + pid_col.append(pids[patient_pos]) + + df["patient_id"] = pid_col + return df diff --git a/v3/scripts/output_analysis/explainability/gradcam_phase5.py b/v3/scripts/output_analysis/explainability/gradcam_phase5.py new file mode 100644 index 0000000..156fb56 --- /dev/null +++ b/v3/scripts/output_analysis/explainability/gradcam_phase5.py @@ -0,0 +1,569 @@ +""" +GradCAM analysis for Phase 5 — logit_mlp_head checkpointed run. + +Produces (all in figures/explainability/gradcam/): + mean_cam_normal.png — average heatmap across all normal test eyes + mean_cam_glaucoma.png — average heatmap across all glaucoma test eyes + mean_cam_comparison.png — side-by-side normal vs glaucoma mean CAMs + overlay_grid_normal.png — grid of individual overlays (normal eyes) + overlay_grid_glaucoma.png — grid of individual overlays (glaucoma eyes) + +Checkpoints loaded from: + v3/results/phase5/logit_mlp_head_ckpt/rep00/binary/ensemble/fold{0..4}/best_single.pt + +Usage: + python -m v3.scripts.output_analysis.explainability.gradcam_phase5 +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.cm as cm +import matplotlib.pyplot as plt +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image +from tqdm import tqdm + +REPO_ROOT = Path(__file__).resolve().parents[4] +CKPT_RUN = REPO_ROOT / "v3" / "results" / "phase5" / "logit_mlp_head_ckpt" +FIGURES_ROOT = REPO_ROOT / "v3" / "figures" / "explainability" / "gradcam" +CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData" +IMAGE_DIR = REPO_ROOT / "Papila" / "FundusImages" +CONTOUR_DIR = REPO_ROOT / "Papila" / "ExpertsSegmentations" / "Contours" + +DISC_SPAN = 5 # patch side = DISC_SPAN × disc diameter +PATCH_SIZE = 96 # output thumbnail pixels + +# Model hyperparameters (inferred from checkpoint weight shapes) +BACKBONE = "resnet50" +NUM_CLASSES = 2 +CD_HIDDEN = 128 +FUSION_DIM = 256 + +LABEL_NAMES = {0: "Normal", 1: "Glaucoma"} + + +# ── GradCAM ────────────────────────────────────────────────────────────────── + +class GradCAM: + """Minimal GradCAM via forward/backward hooks.""" + + def __init__(self, target_layer: torch.nn.Module) -> None: + self._acts = None + self._grads = None + self._h1 = target_layer.register_forward_hook(self._save_acts) + self._h2 = target_layer.register_full_backward_hook(self._save_grads) + + def _save_acts(self, _m, _i, output): + self._acts = output.detach() + + def _save_grads(self, _m, _gi, grad_output): + self._grads = grad_output[0].detach() + + def compute(self, img: torch.Tensor, meta: torch.Tensor, + model: torch.nn.Module, target_class: int | None = None) -> tuple[np.ndarray, int]: + """Return (cam [H,W] normalised 0-1, predicted_class).""" + model.eval() + with torch.enable_grad(): + out = model(img, meta) + pred = int(out.argmax(1).item()) + tc = pred if target_class is None else target_class + model.zero_grad() + out[0, tc].backward() + + weights = self._grads.mean(dim=(2, 3), keepdim=True) + cam = F.relu((weights * self._acts).sum(dim=1, keepdim=True)) + cam = F.interpolate(cam, img.shape[-2:], mode="bilinear", align_corners=False) + cam_np = cam.squeeze().cpu().numpy() + lo, hi = cam_np.min(), cam_np.max() + return (cam_np - lo) / (hi - lo + 1e-8), pred + + def remove(self) -> None: + self._h1.remove(); self._h2.remove() + + +def overlay_gradcam(pil: Image.Image, cam: np.ndarray, alpha: float = 0.45) -> Image.Image: + cam_u8 = (cam * 255).astype(np.uint8) + cam_r = np.array(Image.fromarray(cam_u8).resize(pil.size, Image.BILINEAR)) / 255.0 + colored = (cm.jet(cam_r)[:, :, :3] * 255).astype(np.uint8) + return Image.blend(pil.convert("RGB"), Image.fromarray(colored), alpha) + + +# ── Disc-centred attention helpers ──────────────────────────────────────────── + +def _disc_contour_path(pid: int, eye: str, expert: int = 1) -> Path: + return CONTOUR_DIR / f"RET{pid:03d}{eye}_disc_exp{expert}.txt" + + +def _load_disc_mask(pid: int, eye: str, cam_h: int, cam_w: int) -> np.ndarray | None: + """Load expert disc contour, polygon-fill, resize to (cam_h, cam_w).""" + from PIL import ImageDraw as _ID + p = _disc_contour_path(pid, eye) + if not p.exists(): + return None + try: + arr = np.loadtxt(str(p), dtype=np.float32) + except Exception: + return None + if arr.ndim == 1: + arr = arr.reshape(-1, 2) + if arr.shape[0] < 3: + return None + # Get original image size + img_path = get_image_path(pid, eye) + try: + with Image.open(img_path) as im: + orig_w, orig_h = im.size + except Exception: + return None + canvas = Image.new("L", (orig_w, orig_h), 0) + _ID.Draw(canvas).polygon([tuple(pt) for pt in arr[:, :2]], fill=1) + return np.array(canvas.resize((cam_w, cam_h), Image.NEAREST), dtype=bool) + + +def _disc_centred_patch(cam: np.ndarray, disc_mask: np.ndarray, + span: int = DISC_SPAN, out: int = PATCH_SIZE + ) -> tuple[np.ndarray | None, float | None]: + """Translate+scale cam so disc centroid is centred; return (patch, disc_r_out).""" + if disc_mask is None or disc_mask.sum() == 0: + return None, None + ys, xs = np.where(disc_mask) + cy, cx = ys.mean(), xs.mean() + disc_r = float(np.sqrt(disc_mask.sum() / np.pi)) + half = max(1, int(round(span * disc_r / 2))) + h, w = cam.shape + y0, y1 = int(round(cy)) - half, int(round(cy)) + half + x0, x1 = int(round(cx)) - half, int(round(cx)) + half + pt = max(0, -y0); pb = max(0, y1 - h) + pl = max(0, -x0); pr = max(0, x1 - w) + cam_pad = np.pad(cam, ((pt, pb), (pl, pr)), constant_values=0.0) + patch = cam_pad[y0 + pt: y1 + pt, x0 + pl: x1 + pl] + patch_out = np.array( + Image.fromarray((np.clip(patch, 0, 1) * 255).astype(np.uint8)) + .resize((out, out), Image.BILINEAR) + ) / 255.0 + disc_r_out = out * disc_r / (2 * half) + return patch_out.astype(np.float32), disc_r_out + + +def make_disc_attention_detail( + mean_patches: dict, + stats_rows: list[dict], + out_path: Path, +) -> None: # noqa: C901 + """ + 2-row (Normal / Glaucoma) × 3-col (correct cam | incorrect cam | disc_frac strip). + + mean_patches: {(cls_name, split): (mean_patch_array, mean_disc_r, count)} + stats_rows: list of {true_name, correct, disc_frac} dicts (floats only, no arrays) + """ + import pandas as pd + from matplotlib.patches import Circle + + classes = ["Normal", "Glaucoma"] + splits = ["correct", "incorrect"] + corr_colors = {"correct": "steelblue", "incorrect": "tomato"} + stats = pd.DataFrame(stats_rows) + + # 2 rows (Normal / Glaucoma) × 3 cols (correct cam | incorrect cam | disc_frac strip) + fig, axes = plt.subplots(2, 3, figsize=(13, 8), + gridspec_kw={"width_ratios": [1, 1, 0.75]}) + fig.patch.set_facecolor("#f4f4f4") + + rng = np.random.default_rng(42) + + for ri, cls in enumerate(classes): + # Col 0 & 1: correct / incorrect mean CAMs + for ci, split in enumerate(splits): + ax = axes[ri, ci] + ax.set_facecolor("#222") + key = (cls, split) + if key in mean_patches: + mp, disc_r_out, count = mean_patches[key] + ax.imshow(mp, cmap="jet", vmin=0, vmax=1, origin="upper", + extent=[0, PATCH_SIZE, PATCH_SIZE, 0]) + cx = cy = PATCH_SIZE / 2 + ax.add_patch(Circle((cx, cy), disc_r_out, + fill=False, edgecolor="white", + linewidth=2, linestyle="--")) + ax.set_title(f"{split.capitalize()} (N={count})", fontsize=9) + else: + ax.text(0.5, 0.5, "no data", ha="center", va="center", + transform=ax.transAxes, fontsize=9, color="grey") + ax.set_title(split.capitalize(), fontsize=9) + ax.axis("off") + + # Row label on leftmost column + axes[ri, 0].set_ylabel(cls, fontsize=11, fontweight="bold", labelpad=8) + + # Col 2: disc_frac strip + ax = axes[ri, 2] + ax.set_facecolor("#f4f4f4") + sub = stats[stats["true_name"] == cls].dropna(subset=["disc_frac"]) + for xi, split in enumerate(splits): + pts = sub[sub["correct"] == (split == "correct")]["disc_frac"].values + if len(pts) == 0: + continue + color = corr_colors[split] + jitter = rng.uniform(-0.18, 0.18, size=len(pts)) + ax.scatter(xi + jitter, pts, color=color, alpha=0.7, s=28, edgecolors="none") + ax.hlines(pts.mean(), xi - 0.28, xi + 0.28, colors=color, linewidth=2.5, zorder=5) + ax.set_xticks([0, 1]) + ax.set_xticklabels(["Correct", "Incorrect"], fontsize=9) + ax.set_xlim(-0.55, 1.55) + ax.set_ylim(0, 1) + ax.set_title("Disc fraction", fontsize=9) + ax.grid(axis="y", linestyle="--", alpha=0.3) + if ri == 0: + ax.set_ylabel("Attention mass inside GT disc", fontsize=9) + + fig.suptitle( + "Disc-centred GradCAM attention | dashed circle = GT disc boundary\n" + "Phase 5, logit_mlp_head, fold 0–4", + fontsize=11, fontweight="bold", + ) + fig.tight_layout() + out_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(out_path, dpi=150, bbox_inches="tight") + plt.close(fig) + print(f"Saved: {out_path}") + + +# ── Model loading ───────────────────────────────────────────────────────────── + +def build_model(ckpt_path: Path, device: torch.device): + """Reconstruct SingleEyeHT from checkpoint and load weights.""" + from types import SimpleNamespace + from v3.classes.models import SingleEyeHT + sd = torch.load(ckpt_path, map_location="cpu") + # ClinicalTower only reads clinical_data.feature_dim at init time + cd_in = sd["cd_tower.block0.0.weight"].shape[1] + clinical_shim = SimpleNamespace(feature_dim=cd_in) + model = SingleEyeHT( + backbone=BACKBONE, + freeze_ratio=0.0, + augment=False, + clinical_data=clinical_shim, + num_classes=NUM_CLASSES, + cd_hidden_dim=CD_HIDDEN, + fusion_dim=FUSION_DIM, + ) + model.load_state_dict(sd) + model.to(device).eval() + return model + + +# ── Data helpers ────────────────────────────────────────────────────────────── + +def build_data_bundle(): + """Build the PAPILA DataBundle matching the checkpointed run's feature config.""" + from v3.classes.papila_builders import build_papila_data + import torch as _t + # Auto-detect cd_in from the majority of checkpoints (excludes stale reps). + import collections as _col + all_ckpts = list(CKPT_RUN.glob("rep*/binary/ensemble/fold*/best_single.pt")) + if all_ckpts: + counts = _col.Counter( + _t.load(c, map_location="cpu")["cd_tower.block0.0.weight"].shape[1] + for c in all_ckpts + ) + cd_in = counts.most_common(1)[0][0] + else: + cd_in = 25 + drop_raw = cd_in <= 21 + excl = ["Axial_Length"] if cd_in in (21, 23) else [] + return build_papila_data( + image_dir=str(IMAGE_DIR), + clinical_dir=str(CLINICAL_DIR), + label_col="Diagnosis", + cat_cols=["Gender", "Phakic/Pseudophakic"], + iop_corr_method="ratio", + iop_drop_raw=drop_raw, + exclude_cols=excl, + ) + + +def get_image_path(pid: int, eye: str) -> Path: + return IMAGE_DIR / f"RET{pid:03d}{eye}.jpg" + + +def build_meta_vector(row, data) -> torch.Tensor: + """Build the training-compatible feature vector via DataBundle.vectorize_row.""" + vec = data.vectorize_row(row) + return torch.tensor(vec, dtype=torch.float32).unsqueeze(0) + + +# ── Eval transform ──────────────────────────────────────────────────────────── + +def get_eval_transform(): + from torchvision import transforms + return transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225)), + ]) + + +# ── Main loop ───────────────────────────────────────────────────────────────── + +def _discover_checkpoints(ckpt_run: Path) -> list[tuple[int, int, Path]]: + """ + Scan ckpt_run for all available best_single.pt files. + Skips checkpoints whose cd_in doesn't match the majority (to exclude stale reps). + Returns sorted list of (rep_idx, fold_idx, ckpt_path). + """ + import collections + candidates = [] + for rep_dir in sorted(ckpt_run.glob("rep*")): + try: + rep_idx = int(rep_dir.name.replace("rep", "")) + except ValueError: + continue + for fold_dir in sorted((rep_dir / "binary" / "ensemble").glob("fold[0-9]")): + ckpt = fold_dir / "best_single.pt" + if ckpt.exists(): + fold_idx = int(fold_dir.name.replace("fold", "")) + cd_in = torch.load(ckpt, map_location="cpu")[ + "cd_tower.block0.0.weight" + ].shape[1] + candidates.append((rep_idx, fold_idx, ckpt, cd_in)) + + if not candidates: + return [] + + # Use the majority cd_in so stale reps are automatically excluded + counts = collections.Counter(c[3] for c in candidates) + target_cd_in = counts.most_common(1)[0][0] + skipped = sum(1 for c in candidates if c[3] != target_cd_in) + if skipped: + print(f" [discover] skipping {skipped} checkpoint(s) with cd_in≠{target_cd_in}") + + return [(rep, fold, ckpt) for rep, fold, ckpt, cd in candidates if cd == target_cd_in] + + +def run(n_grid: int = 16, alpha: float = 0.45, target_class: int | None = None): + """ + Loop over all available checkpoints in the run directory (all reps × folds). + Aggregate CAMs per class, collect overlay grids. + """ + import pandas as pd + from v3.scripts.output_analysis.explainability.fold_patient_ids import ( + get_test_patient_ids, + ) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + print(f"Device: {device}") + + print("Building DataBundle ...") + data = build_data_bundle() + clinical = data.df + print(f" feature_dim={data.feature_dim} rows={len(clinical)}") + transform = get_eval_transform() + + checkpoints = _discover_checkpoints(CKPT_RUN) + print(f"Found {len(checkpoints)} checkpoint(s) across " + f"{len(set(r for r,f,_ in checkpoints))} rep(s)") + + if not checkpoints: + print("No checkpoints found — run with --save-checkpoints first.") + return + + # ── Incremental accumulators (no full-res arrays kept after each eye) ──────── + # Mean CAM per class: running sum + cam_sum = {0: None, 1: None} + cam_count = {0: 0, 1: 0} + + # Overlay grid: keep at most n_grid PIL images per class (capped) + overlay_items = {0: [], 1: []} + + # Disc-attention detail: running sum of disc patches (not list of arrays) + disc_patch_sum = {} # (cls_name, split) → np.ndarray sum + disc_patch_count = {} # (cls_name, split) → int + disc_radius_sum = {} # (cls_name, split) → float sum + disc_stats_rows = [] # floats only — no arrays + + ckpt_bar = tqdm(checkpoints, desc="Folds", unit="fold") + for rep_idx, fold_idx, ckpt_path in ckpt_bar: + ckpt_bar.set_postfix(rep=rep_idx, fold=fold_idx) + model = build_model(ckpt_path, device) + + # GradCAM target: last ResNet block + target_layer = model.img_tower.backbone.layer4[-1] + gcam = GradCAM(target_layer) + + pids = get_test_patient_ids(rep_idx, fold_idx, clinical_dir=CLINICAL_DIR) + + for pid in tqdm(pids, desc=f" rep{rep_idx:02d}/fold{fold_idx}", leave=False, unit="pt"): + for eye in ("OD", "OS"): + img_path = get_image_path(pid, eye) + if not img_path.exists(): + continue + + row = clinical[ + (clinical["Patient ID"] == pid) & (clinical["eyeID"] == eye) + ] + if len(row) == 0: + continue + row = row.iloc[0] + label = int(row["Diagnosis"]) + + pil_orig = Image.open(img_path).convert("RGB") + img_t = transform(pil_orig).unsqueeze(0).to(device) + meta_t = build_meta_vector(row, data).to(device) + + cam_np, pred = gcam.compute(img_t, meta_t, model, + target_class=target_class) + + # Running mean CAM + if cam_sum[label] is None: + cam_sum[label] = cam_np.copy() + else: + cam_sum[label] += cam_np + cam_count[label] += 1 + + # Overlay grid — only keep up to n_grid per class + if len(overlay_items[label]) < n_grid: + ov = overlay_gradcam(pil_orig, cam_np, alpha=alpha) + overlay_items[label].append((ov, pid, eye, pred)) + + # Disc-attention: extract patch now, accumulate into running sum + h, w = cam_np.shape + disc_mask = _load_disc_mask(pid, eye, h, w) + disc_frac = None + if disc_mask is not None and disc_mask.sum() > 0: + disc_frac = float(cam_np[disc_mask].sum() / (cam_np.sum() + 1e-8)) + + cls_name = LABEL_NAMES[label] + split = "correct" if (pred == label) else "incorrect" + key = (cls_name, split) + + patch, disc_r_out = _disc_centred_patch(cam_np, disc_mask) + if patch is not None: + if key not in disc_patch_sum: + disc_patch_sum[key] = patch.copy() + disc_patch_count[key] = 1 + disc_radius_sum[key] = disc_r_out + else: + disc_patch_sum[key] += patch + disc_patch_count[key] += 1 + disc_radius_sum[key] += disc_r_out + + disc_stats_rows.append({ + "true_name": cls_name, + "correct": pred == label, + "disc_frac": disc_frac, + }) + + # Release per-eye tensors immediately + del img_t, meta_t, cam_np + if disc_mask is not None: + del disc_mask + + gcam.remove() + del model + torch.cuda.empty_cache() if torch.cuda.is_available() else None + + # Build mean_patches dict for disc detail plot + mean_patches = { + key: ( + disc_patch_sum[key] / disc_patch_count[key], + disc_radius_sum[key] / disc_patch_count[key], + disc_patch_count[key], + ) + for key in disc_patch_sum + } + + # ── Save outputs ───────────────────────────────────────────────────────── + FIGURES_ROOT.mkdir(parents=True, exist_ok=True) + + for cls in [0, 1]: + if cam_count[cls] == 0: + continue + mean_cam = cam_sum[cls] / cam_count[cls] + lo, hi = mean_cam.min(), mean_cam.max() + mean_cam = (mean_cam - lo) / (hi - lo + 1e-8) + + fig, ax = plt.subplots(figsize=(5, 5)) + ax.imshow(mean_cam, cmap="jet", vmin=0, vmax=1) + ax.axis("off") + ax.set_title(f"Mean GradCAM — {LABEL_NAMES[cls]}\n(n={cam_count[cls]} eyes, fold 0–4)", + fontsize=11, fontweight="bold") + plt.colorbar(ax.images[0], ax=ax, fraction=0.046, pad=0.04) + out = FIGURES_ROOT / f"mean_cam_{LABEL_NAMES[cls].lower()}.png" + fig.savefig(out, dpi=180, bbox_inches="tight") + plt.close(fig) + print(f"Saved: {out}") + + # Side-by-side comparison + if cam_count[0] > 0 and cam_count[1] > 0: + fig, axes = plt.subplots(1, 2, figsize=(10, 5)) + fig.suptitle("Mean GradCAM — Normal vs Glaucoma (Phase 5, fold 0–4)", + fontsize=12, fontweight="bold") + for ax, cls in zip(axes, [0, 1]): + mean_cam = cam_sum[cls] / cam_count[cls] + lo, hi = mean_cam.min(), mean_cam.max() + mean_cam = (mean_cam - lo) / (hi - lo + 1e-8) + im = ax.imshow(mean_cam, cmap="jet", vmin=0, vmax=1) + ax.axis("off") + ax.set_title(f"{LABEL_NAMES[cls]} (n={cam_count[cls]})", fontsize=11) + plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04) + out = FIGURES_ROOT / "mean_cam_comparison.png" + fig.savefig(out, dpi=180, bbox_inches="tight") + plt.close(fig) + print(f"Saved: {out}") + + # Overlay grids + for cls in [0, 1]: + items = overlay_items[cls] + if not items: + continue + # Sort: misclassified first (more interesting) + items.sort(key=lambda x: x[3] == cls) # wrong preds first + items = items[:n_grid] + ncols = 4 + nrows = int(np.ceil(len(items) / ncols)) + fig, axes = plt.subplots(nrows, ncols, figsize=(ncols * 3.2, nrows * 3.2)) + axes = np.array(axes).reshape(-1) + fig.suptitle(f"GradCAM Overlays — {LABEL_NAMES[cls]} (Phase 5)", + fontsize=12, fontweight="bold") + for i, ax in enumerate(axes): + if i < len(items): + ov, pid, eye, pred = items[i] + ax.imshow(ov) + correct = pred == cls + col = "#2e7d32" if correct else "#c62828" + ax.set_title(f"RET{pid:03d}{eye}\n→ {LABEL_NAMES[pred]}", + fontsize=7.5, color=col) + ax.axis("off") + out = FIGURES_ROOT / f"overlay_grid_{LABEL_NAMES[cls].lower()}.png" + fig.tight_layout() + fig.savefig(out, dpi=150, bbox_inches="tight") + plt.close(fig) + print(f"Saved: {out}") + + + # Disc-centred detail plot + if disc_patch_sum: + make_disc_attention_detail(mean_patches, disc_stats_rows, + FIGURES_ROOT / "disc_attention_detail.png") + + +if __name__ == "__main__": + import argparse + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--n-grid", type=int, default=16, + help="Max overlays per class in grid (default 16)") + ap.add_argument("--alpha", type=float, default=0.45, + help="GradCAM overlay opacity (default 0.45)") + ap.add_argument("--target-class", type=int, default=None, + help="GradCAM target class (default: predicted class)") + args = ap.parse_args() + run(n_grid=args.n_grid, alpha=args.alpha, target_class=args.target_class) diff --git a/v3/scripts/output_analysis/explainability/permutation_importance_phase5.py b/v3/scripts/output_analysis/explainability/permutation_importance_phase5.py new file mode 100644 index 0000000..a4c4601 --- /dev/null +++ b/v3/scripts/output_analysis/explainability/permutation_importance_phase5.py @@ -0,0 +1,309 @@ +""" +MD permutation feature importance for Phase 5 — logit_mlp_head checkpointed run. + +For each of the 5 fold checkpoints: + - loads test images + clinical metadata + - caches image features (no grad) + - permutes each clinical feature N times and measures AUC drop + +Produces (in figures/explainability/): + md_importance_phase5.png — aggregated bar chart across 5 folds + md_importance_phase5.csv — mean/std per feature + +Usage: + python -m v3.scripts.output_analysis.explainability.permutation_importance_phase5 +""" +from __future__ import annotations + +import csv +from pathlib import Path +from types import SimpleNamespace + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import torch +import torch.nn.functional as F +from sklearn.metrics import roc_auc_score + +REPO_ROOT = Path(__file__).resolve().parents[4] +CKPT_RUN = REPO_ROOT / "v3" / "results" / "phase5" / "logit_mlp_head_ckpt" +FIGURES_ROOT = REPO_ROOT / "v3" / "figures" / "explainability" +CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData" +IMAGE_DIR = REPO_ROOT / "Papila" / "FundusImages" + +BACKBONE = "resnet50" +NUM_CLASSES = 2 +CD_HIDDEN = 128 +FUSION_DIM = 256 +N_PERMUTATIONS = 30 +SEED = 0 + + +# ── Model ───────────────────────────────────────────────────────────────────── + +def build_model(ckpt_path: Path, device: torch.device): + from v3.classes.models import SingleEyeHT + sd = torch.load(ckpt_path, map_location="cpu") + cd_in = sd["cd_tower.block0.0.weight"].shape[1] + model = SingleEyeHT( + backbone=BACKBONE, freeze_ratio=0.0, augment=False, + clinical_data=SimpleNamespace(feature_dim=cd_in), + num_classes=NUM_CLASSES, cd_hidden_dim=CD_HIDDEN, fusion_dim=FUSION_DIM, + ) + model.load_state_dict(sd) + model.to(device).eval() + return model + + +# ── Data ────────────────────────────────────────────────────────────────────── + +def build_data_bundle(): + from v3.classes.papila_builders import build_papila_data + # Infer settings from available checkpoint to stay compatible. + # Once the 10x5 run (--iop-drop-raw --exclude-cols Axial_Length) completes, + # these will automatically match (feature_dim will drop from 25 → 21). + ckpt = next(CKPT_RUN.glob("rep*/binary/ensemble/fold*/best_single.pt"), None) + import torch as _t + cd_in = _t.load(ckpt, map_location="cpu")["cd_tower.block0.0.weight"].shape[1] if ckpt else 25 + # cd_in=25 → old run (no iop_drop_raw, no excl); cd_in=21 → new run + drop_raw = cd_in <= 21 + excl = ["Axial_Length"] if cd_in in (21, 23) else [] + return build_papila_data( + image_dir=str(IMAGE_DIR), clinical_dir=str(CLINICAL_DIR), + label_col="Diagnosis", cat_cols=["Gender", "Phakic/Pseudophakic"], + iop_corr_method="ratio", iop_drop_raw=drop_raw, exclude_cols=excl, + ) + + +def get_eval_transform(): + from torchvision import transforms + return transforms.Compose([ + transforms.Resize(256), transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), + ]) + + +def get_image_path(pid: int, eye: str) -> Path: + return IMAGE_DIR / f"RET{pid:03d}{eye}.jpg" + + +# ── Feature index map ───────────────────────────────────────────────────────── + +def build_feature_index_map(data) -> dict[str, list[int]]: + """ + Map feature name → list of dimension indices in the vectorize_row output. + Layout: [scalars (min-max scaled)] + [cat one-hots] + [scalar missing flags] + """ + n_scalar = len(data.scalar_cols) + cat_expanded = sum(len(m) for m in data.cat_maps.values()) + feat_map: dict[str, list[int]] = {} + + # Scalar: value dim + missing flag dim + for i, col in enumerate(data.scalar_cols): + feat_map[col] = [i, n_scalar + cat_expanded + i] + + # Categorical: whole one-hot block + cat_offset = n_scalar + for col in data.cat_cols: + n = len(data.cat_maps[col]) + feat_map[col] = list(range(cat_offset, cat_offset + n)) + cat_offset += n + + return feat_map + + +# ── Per-fold importance ─────────────────────────────────────────────────────── + +def run_fold(rep_idx: int, fold_idx: int, model, data, device: torch.device, + n_permutations: int, seed: int) -> dict[str, tuple[float, float]]: + """ + Returns {feature_name: (mean_auc_drop, std_auc_drop)}. + """ + from v3.scripts.output_analysis.explainability.fold_patient_ids import ( + get_test_patient_ids, + ) + transform = get_eval_transform() + clinical = data.df + + pids = get_test_patient_ids(rep_idx, fold_idx, clinical_dir=CLINICAL_DIR) + + # Cache image features + build meta tensors + labels + img_feats_list, meta_list, label_list = [], [], [] + model.eval() + with torch.no_grad(): + for pid in pids: + for eye in ("OD", "OS"): + img_path = get_image_path(pid, eye) + if not img_path.exists(): + continue + row = clinical[ + (clinical["Patient ID"] == pid) & (clinical["eyeID"] == eye) + ] + if len(row) == 0: + continue + row = row.iloc[0] + label = int(row["Diagnosis"]) + + from PIL import Image + pil = Image.open(img_path).convert("RGB") + img_t = transform(pil).unsqueeze(0).to(device) + feats = model.img_tower(img_t) # [1, img_dim] + meta_vec = torch.tensor(data.vectorize_row(row), + dtype=torch.float32).unsqueeze(0) + + img_feats_list.append(feats.cpu()) + meta_list.append(meta_vec) + label_list.append(label) + + if not label_list or len(set(label_list)) < 2: + print(f" fold{fold_idx}: insufficient data, skipping.") + return {} + + img_feats = torch.cat(img_feats_list).to(device) # [N, img_dim] + meta_all = torch.cat(meta_list) # [N, feat_dim] on CPU + y_true = np.array(label_list) + + # Baseline AUC + with torch.no_grad(): + md_feats = model.cd_tower(meta_all.to(device)) + out_f, _, _ = model.bridge(img_feats, md_feats) + probs_base = F.softmax(out_f, dim=1)[:, 1].cpu().numpy() + baseline_auc = roc_auc_score(y_true, probs_base) + print(f" fold{fold_idx}: baseline AUC={baseline_auc:.4f} N={len(y_true)}") + + feat_map = build_feature_index_map(data) + rng = np.random.default_rng(seed + fold_idx) + results: dict[str, tuple[float, float]] = {} + + for feat_name, dims in feat_map.items(): + drops = [] + for _ in range(n_permutations): + meta_perm = meta_all.clone() + perm_idx = rng.permutation(len(meta_perm)) + meta_perm[:, dims] = meta_perm[perm_idx][:, dims] + with torch.no_grad(): + md_p = model.cd_tower(meta_perm.to(device)) + out_p, _, _ = model.bridge(img_feats, md_p) + probs_p = F.softmax(out_p, dim=1)[:, 1].cpu().numpy() + try: + drops.append(baseline_auc - roc_auc_score(y_true, probs_p)) + except Exception: + pass + if drops: + results[feat_name] = (float(np.mean(drops)), float(np.std(drops))) + + return results + + +# ── Aggregate and plot ──────────────────────────────────────────────────────── + +def plot_importance(all_results: list[dict], out_png: Path, out_csv: Path) -> None: + # Aggregate across folds + all_feats = sorted({f for r in all_results for f in r}) + agg = {} + for feat in all_feats: + vals = [r[feat][0] for r in all_results if feat in r] + if vals: + agg[feat] = (float(np.mean(vals)), float(np.std(vals))) + + # Sort by mean importance descending + sorted_feats = sorted(agg, key=lambda f: agg[f][0], reverse=True) + names = sorted_feats + imps = [agg[f][0] for f in names] + stds = [agg[f][1] for f in names] + colors = ["#e05c5c" if v >= 0 else "#5c9ee0" for v in imps] + + fig, ax = plt.subplots(figsize=(9, max(4, len(names) * 0.45 + 1.5))) + y_pos = np.arange(len(names)) + ax.barh(y_pos, imps, xerr=stds, color=colors, ecolor="grey", capsize=3, height=0.6) + ax.set_yticks(y_pos) + ax.set_yticklabels(names, fontsize=9) + ax.invert_yaxis() + ax.axvline(0, color="black", linewidth=0.8) + ax.set_xlabel("Mean AUC drop (baseline − permuted)", fontsize=10) + n_reps = len(set(r for r in range(len(all_results)))) # placeholder + ax.set_title( + f"MD Tower — Permutation Feature Importance\n" + f"Phase 5 logit_mlp_head_ckpt ({len(all_results)} folds, " + f"error bars = std across folds)", + fontsize=11, + ) + fig.tight_layout() + out_png.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(out_png, dpi=150, bbox_inches="tight") + plt.close(fig) + print(f"Saved: {out_png}") + + with open(out_csv, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=["feature", "mean_importance", "std_importance"]) + w.writeheader() + for feat in sorted_feats: + w.writerow({"feature": feat, + "mean_importance": agg[feat][0], + "std_importance": agg[feat][1]}) + print(f"Saved: {out_csv}") + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def _discover_checkpoints(ckpt_run: Path) -> list[tuple[int, int, Path]]: + found = [] + for rep_dir in sorted(ckpt_run.glob("rep*")): + try: + rep_idx = int(rep_dir.name.replace("rep", "")) + except ValueError: + continue + for fold_dir in sorted((rep_dir / "binary" / "ensemble").glob("fold[0-9]")): + ckpt = fold_dir / "best_single.pt" + if ckpt.exists(): + found.append((rep_idx, int(fold_dir.name.replace("fold", "")), ckpt)) + return found + + +def main(n_permutations: int = N_PERMUTATIONS, seed: int = SEED): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + print(f"Device: {device}") + + print("Building DataBundle ...") + data = build_data_bundle() + print(f" feature_dim={data.feature_dim}") + + checkpoints = _discover_checkpoints(CKPT_RUN) + print(f"Found {len(checkpoints)} checkpoint(s) across " + f"{len(set(r for r,f,_ in checkpoints))} rep(s)") + + if not checkpoints: + print("No checkpoints found.") + return + + all_results = [] + for rep_idx, fold_idx, ckpt in checkpoints: + print(f"\n── rep{rep_idx:02d} fold{fold_idx} ──") + model = build_model(ckpt, device) + result = run_fold(rep_idx, fold_idx, model, data, device, n_permutations, seed) + if result: + all_results.append(result) + del model + + if not all_results: + print("No results — nothing to plot.") + return + + plot_importance( + all_results, + FIGURES_ROOT / "md_importance_phase5.png", + FIGURES_ROOT / "md_importance_phase5.csv", + ) + + +if __name__ == "__main__": + import argparse + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--n-permutations", type=int, default=N_PERMUTATIONS) + ap.add_argument("--seed", type=int, default=SEED) + args = ap.parse_args() + main(n_permutations=args.n_permutations, seed=args.seed) diff --git a/v3/scripts/output_analysis/plot_architecture.py b/v3/scripts/output_analysis/plot_architecture.py new file mode 100644 index 0000000..b8eb633 --- /dev/null +++ b/v3/scripts/output_analysis/plot_architecture.py @@ -0,0 +1,693 @@ +""" +Publication-quality architecture diagrams for HyperTower. + +Generates: + architecture_single_tower.png — single-eye image-only tower + architecture_hypertower.png — single-eye image + clinical fusion + architecture_ensemble.png — bilateral ensemble (two HyperTowers + average) + architecture_fused_head.png — bilateral ensemble + learned head + +Usage: + python -m v3.scripts.output_analysis.plot_architecture + python -m v3.scripts.output_analysis.plot_architecture --out figures/ +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from matplotlib.patches import FancyBboxPatch, FancyArrowPatch +import matplotlib.patheffects as pe + +# ── Colour palette ──────────────────────────────────────────────────────────── +C_IMG = "#4e8d3a" # green — image / CNN +C_MD = "#4c72b0" # blue — clinical / MLP +C_BRIDGE = "#c44e52" # red — bridge / fusion +C_EMB = "#2a9d8f" # teal — embedding vectors (z) +C_OUT = "#8c6bb1" # purple — output nodes +C_HEAD = "#d4a017" # gold — learned head / average +C_INPUT = "#a0a0a0" # grey — raw input nodes +C_BG = "#e8e8e8" +C_ARROW = "#444444" +FONT = "DejaVu Sans" + + +# ── Low-level primitives ────────────────────────────────────────────────────── + + +def _box( + ax, + cx, + cy, + w, + h, + color, + text="", + fontsize=9, + text_color="white", + bold=False, + alpha=0.92, + radius=0.12, + lw=1.5, +): + """Rounded rectangle centered at (cx, cy).""" + patch = FancyBboxPatch( + (cx - w / 2, cy - h / 2), + w, + h, + boxstyle=f"round,pad=0,rounding_size={radius}", + facecolor=color, + edgecolor="white", + linewidth=lw, + alpha=alpha, + zorder=3, + transform=ax.transData, + ) + ax.add_patch(patch) + if text: + ax.text( + cx, + cy, + text, + ha="center", + va="center", + fontsize=fontsize, + color=text_color, + fontweight="bold" if bold else "normal", + fontfamily=FONT, + zorder=4, + ) + return patch + + +def _arrow(ax, x0, y0, x1, y1, lw=1.6, color=C_ARROW, style="->", rad=0.0): + ax.annotate( + "", + xy=(x1, y1), + xytext=(x0, y0), + arrowprops=dict( + arrowstyle=style, + color=color, + lw=lw, + connectionstyle=f"arc3,rad={rad}", + ), + zorder=2, + ) + + +def _text( + ax, x, y, s, fontsize=8, color="#333333", ha="center", va="center", bold=False +): + ax.text( + x, + y, + s, + ha=ha, + va=va, + fontsize=fontsize, + color=color, + fontfamily=FONT, + fontweight="bold" if bold else "normal", + zorder=5, + ) + + +def _bracket( + ax, + x, + y0, + y1, + text="", + fontsize=8.5, + color="#888888", + pad=0.15, + lw=1.4, + badge_color=None, +): + """Vertical C-bracket on the right side. + If badge_color is set, the label is drawn as white text on a filled badge.""" + mid = (y0 + y1) / 2 + ax.plot( + [x, x + pad, x + pad, x], + [y1, y1, y0, y0], + color=color, + lw=lw, + solid_capstyle="round", + zorder=2, + ) + if text: + if badge_color: + ax.text( + x + pad * 1.4, + mid, + text, + ha="left", + va="center", + fontsize=fontsize, + color="white", + fontfamily=FONT, + fontweight="bold", + zorder=6, + bbox=dict( + facecolor=badge_color, + edgecolor="none", + pad=3.5, + boxstyle="round,pad=0.3", + ), + ) + else: + ax.text( + x + pad * 1.4, + mid, + text, + ha="left", + va="center", + fontsize=fontsize, + color=color, + fontfamily=FONT, + style="italic", + ) + + +def _setup(fig, ax, w, h, title): + ax.set_xlim(0, w) + ax.set_ylim(0, h) + ax.axis("off") + ax.set_facecolor(C_BG) + fig.patch.set_facecolor(C_BG) + if title: + ax.set_title( + title, + fontsize=12, + fontweight="bold", + fontfamily=FONT, + pad=10, + color="#222222", + ) + + +# ── Reusable sub-blocks ─────────────────────────────────────────────────────── + + +def _draw_cnn_block(ax, x_center, y, w=2.0, h=0.75): + """Three-layer CNN block with labels: Conv Layers → Conv Layers → GAP.""" + labels = ["Conv\nLayers", "Conv\nLayers", "GAP"] + sub_w = [w * 0.42, w * 0.30, w * 0.22] + sub_h = [h, h * 0.82, h * 0.65] + alphas = [0.82, 0.74, 0.66] + fsizes = [8.0, 7.5, 7.5] + gap = (w - sum(sub_w)) / 2 + xs = [ + x_center - w / 2 + sub_w[0] / 2, + x_center - w / 2 + sub_w[0] + gap + sub_w[1] / 2, + x_center - w / 2 + sub_w[0] + gap + sub_w[1] + gap + sub_w[2] / 2, + ] + for i, (sx, sw, sh, lbl, alp, fs) in enumerate( + zip(xs, sub_w, sub_h, labels, alphas, fsizes) + ): + _box(ax, sx, y, sw, sh, C_IMG, lbl, fontsize=fs, alpha=alp, radius=0.08) + if i < 2: + _arrow( + ax, sx + sw / 2, y, xs[i + 1] - sub_w[i + 1] / 2, y, lw=1.2, style="-|>" + ) + return xs[-1] + sub_w[-1] / 2 + + +def _draw_mlp_block(ax, x_center, y, w=1.4, h=0.65): + """Two-layer MLP block: FC(128) → FC(128) (hidden_dim=128 both layers).""" + labels = ["FC\n(128)", "FC\n(128)"] + w0, w1 = w * 0.55, w * 0.45 + gap = w - w0 - w1 + x0 = x_center - w / 2 + w0 / 2 + x1 = x0 + w0 / 2 + gap + w1 / 2 + _box(ax, x0, y, w0, h, C_MD, labels[0], fontsize=8.0, alpha=0.82, radius=0.08) + _arrow(ax, x0 + w0 / 2, y, x1 - w1 / 2, y, lw=1.2, style="-|>") + _box( + ax, x1, y, w1, h * 0.88, C_MD, labels[1], fontsize=7.5, alpha=0.72, radius=0.08 + ) + return x1 + w1 / 2 + + +def _draw_embedding(ax, x, y, w=0.40, h=0.75, label="z\n(emb)"): + _box(ax, x + w / 2, y, w, h, C_EMB, label, fontsize=8, bold=True, radius=0.08) + return x + w + + +def _draw_output(ax, x, y, dy=0.45, classes=("Glaucoma", "Normal")): + """Stacked output class boxes, connected from (x, y) via arrows.""" + n = len(classes) + bw = 1.10 + bh = 0.38 + gap = 0.08 + total = n * bh + (n - 1) * gap + y_top = y + total / 2 - bh / 2 + + for i, cls in enumerate(classes): + cy = y_top - i * (bh + gap) + _box(ax, x + bw / 2, cy, bw, bh, C_OUT, cls, fontsize=8.5, radius=0.08) + _arrow(ax, x, y, x, cy, lw=1.1, style="-|>", rad=0.0) + + _text(ax, x + bw / 2, y - total / 2 - 0.20, "Softmax", fontsize=7.5, color=C_OUT) + + +def _draw_compact_ht(ax, x_left, y_img, y_md, eye_label): + """Compact HyperTower block: Image+Clinical boxes → Bridge. + Returns (x_right_of_bridge, y_bridge_center). + """ + bw_img = 1.40 + bh_img = 0.72 + bw_md = 1.20 + bh_md = 0.62 + bw_br = 0.72 + cy_br = (y_img + y_md) / 2 + bh_br = abs(y_img - y_md) * 0.60 + + # Image box: CNN Backbone + _box( + ax, + x_left + bw_img / 2, + y_img, + bw_img, + bh_img, + C_IMG, + f"{eye_label}\nCNN Backbone", + fontsize=8.5, + radius=0.08, + ) + # MD box: Clinical MLP + _box( + ax, + x_left + bw_md / 2, + y_md, + bw_md, + bh_md, + C_MD, + f"{eye_label}\nClinical MLP", + fontsize=8.5, + radius=0.08, + ) + + # Arrows to bridge + br_x = x_left + max(bw_img, bw_md) + 0.60 + _arrow(ax, x_left + bw_img, y_img, br_x - bw_br / 2, cy_br, lw=1.2, style="-|>") + _arrow(ax, x_left + bw_md, y_md, br_x - bw_br / 2, cy_br, lw=1.2, style="-|>") + + # Bridge label kept simple — detail lives in the hypertower diagram + _box( + ax, + br_x, + cy_br, + bw_br, + max(bh_br, 0.70), + C_BRIDGE, + "Bridge", + fontsize=8.0, + radius=0.08, + ) + + return br_x + bw_br / 2, cy_br + + +# ── Figure 1: Single Tower ──────────────────────────────────────────────────── + + +def make_single_tower(out_dir: Path): + W, H = 9.0, 3.2 + fig, ax = plt.subplots(figsize=(W, H)) + _setup(fig, ax, W, H, "Single Tower") + + cy = H / 2 + + # Input + _box( + ax, + 0.75, + cy, + 0.95, + 0.60, + C_INPUT, + "Fundus\nImage", + fontsize=8.5, + radius=0.08, + alpha=0.75, + text_color="#333", + ) + _arrow(ax, 1.22, cy, 1.60, cy) + + # CNN Backbone + cnn_x_right = _draw_cnn_block(ax, x_center=3.10, y=cy, w=2.80, h=0.78) + _text(ax, 3.10, cy - 0.68, "CNN Backbone", fontsize=8.5, color=C_IMG, bold=True) + _arrow(ax, 1.60, cy, 1.73, cy, lw=1.4, style="-|>") + + # Embedding + emb_x_right = _draw_embedding(ax, x=cnn_x_right + 0.28, y=cy, w=0.48, h=0.78) + _arrow(ax, cnn_x_right, cy, cnn_x_right + 0.28, cy, lw=1.4, style="-|>") + + # Classifier + _arrow(ax, emb_x_right, cy, emb_x_right + 0.25, cy, lw=1.4, style="-|>") + _draw_output(ax, emb_x_right + 0.25, cy) + + path = out_dir / "architecture_single_tower.png" + fig.savefig(path, dpi=180, bbox_inches="tight") + plt.close(fig) + print(f" Saved: {path}") + + +# ── Figure 2: HyperTower (single eye) ──────────────────────────────────────── + + +def make_hypertower(out_dir: Path): + W, H = 11.0, 5.5 + fig, ax = plt.subplots(figsize=(W, H)) + _setup(fig, ax, W, H, "HyperTower — Single Eye") + + y_img = 3.70 + y_md = 1.60 + + # ── Image tower ──────────────────────────────────────────────── + _box( + ax, + 0.80, + y_img, + 1.00, + 0.60, + C_INPUT, + "Fundus\nImage", + fontsize=8.5, + radius=0.08, + alpha=0.75, + text_color="#333", + ) + _arrow(ax, 1.30, y_img, 1.85, y_img) + cnn_x_r = _draw_cnn_block(ax, x_center=3.50, y=y_img, w=2.80, h=0.75) + _text(ax, 3.50, y_img - 0.65, "CNN Backbone", fontsize=8, color=C_IMG, bold=True) + _arrow(ax, 1.85, y_img, 1.98, y_img, lw=1.4, style="-|>") + + emb_img_x = _draw_embedding(ax, x=cnn_x_r + 0.30, y=y_img, w=0.65, h=0.75) + _arrow(ax, cnn_x_r, y_img, cnn_x_r + 0.30, y_img, lw=1.4, style="-|>") + _text( + ax, + (1.30 + emb_img_x) / 2, + y_img + 0.65, + "Image Tower", + fontsize=9, + color=C_IMG, + bold=True, + ) + + # ── Clinical tower ───────────────────────────────────────────── + _box( + ax, + 0.80, + y_md, + 1.00, + 0.55, + C_INPUT, + "Clinical\nData", + fontsize=8.5, + radius=0.08, + alpha=0.75, + text_color="#333", + ) + _arrow(ax, 1.30, y_md, 1.65, y_md) + mlp_x_r = _draw_mlp_block(ax, x_center=2.90, y=y_md, w=1.60, h=0.65) + _arrow(ax, 1.65, y_md, 1.74, y_md, lw=1.4, style="-|>") + + emb_md_x = _draw_embedding(ax, x=mlp_x_r + 0.30, y=y_md, w=0.65, h=0.65) + _arrow(ax, mlp_x_r, y_md, mlp_x_r + 0.30, y_md, lw=1.4, style="-|>") + _text( + ax, + (1.30 + emb_md_x) / 2, + y_md - 0.60, + "Clinical Tower", + fontsize=9, + color=C_MD, + bold=True, + ) + + # ── Bridge ───────────────────────────────────────────────────── + br_x = max(emb_img_x, emb_md_x) + 0.80 + cy_br = (y_img + y_md) / 2 + bh_br = abs(y_img - y_md) * 0.55 + + _arrow(ax, emb_img_x, y_img, br_x - 0.40, cy_br, lw=1.4, style="-|>") + _arrow(ax, emb_md_x, y_md, br_x - 0.40, cy_br, lw=1.4, style="-|>") + _box( + ax, + br_x, + cy_br, + 1.40, + max(bh_br, 1.35), + C_BRIDGE, + "Bridge\nFC(img→256)\nFC(md→256)\n⊙ Hadamard\n→ ReLU→FC(2)", + fontsize=8, + radius=0.10, + ) + + # ── Output ───────────────────────────────────────────────────── + out_x = br_x + 0.65 + 0.40 + _arrow(ax, br_x + 0.65, cy_br, out_x, cy_br, lw=1.4, style="-|>") + _draw_output(ax, out_x, cy_br) + + # ── Bracket (right of output nodes; output bw=1.10 so right edge = out_x+1.10) + _bracket( + ax, + x=out_x + 1.25, + y0=y_md - 0.50, + y1=y_img + 0.50, + text="HyperTower", + fontsize=9, + pad=0.22, + color="#555", + badge_color="#555", + ) + + path = out_dir / "architecture_hypertower.png" + fig.savefig(path, dpi=180, bbox_inches="tight") + plt.close(fig) + print(f" Saved: {path}") + + +# ── Figure 3: Bilateral Ensemble ───────────────────────────────────────────── + + +def make_ensemble(out_dir: Path): + W, H = 10.5, 7.5 + fig, ax = plt.subplots(figsize=(W, H)) + _setup(fig, ax, W, H, "Bilateral Ensemble HyperTower") + + x_left = 3.0 + inp_cx = 1.85 + inp_w = 0.90 + inp_h = 0.55 + + # OD (top) + od_y_img, od_y_md = 5.90, 4.60 + br_od_x, cy_od = _draw_compact_ht( + ax, x_left=x_left, y_img=od_y_img, y_md=od_y_md, eye_label="OD" + ) + _text(ax, 0.45, (od_y_img + od_y_md) / 2, "OD\n(Right Eye)", + fontsize=9, color="#444", bold=True) + _box(ax, inp_cx, od_y_img, inp_w, inp_h, C_INPUT, "Fundus\nImage", + fontsize=8, radius=0.08, alpha=0.75, text_color="#333") + _box(ax, inp_cx, od_y_md, inp_w, inp_h, C_INPUT, "Clinical\nData", + fontsize=8, radius=0.08, alpha=0.75, text_color="#333") + _arrow(ax, inp_cx + inp_w / 2, od_y_img, x_left, od_y_img, lw=1.2, style="-|>") + _arrow(ax, inp_cx + inp_w / 2, od_y_md, x_left, od_y_md, lw=1.2, style="-|>") + + # OS (bottom) + os_y_img, os_y_md = 2.80, 1.50 + br_os_x, cy_os = _draw_compact_ht( + ax, x_left=x_left, y_img=os_y_img, y_md=os_y_md, eye_label="OS" + ) + _text(ax, 0.45, (os_y_img + os_y_md) / 2, "OS\n(Left Eye)", + fontsize=9, color="#444", bold=True) + _box(ax, inp_cx, os_y_img, inp_w, inp_h, C_INPUT, "Fundus\nImage", + fontsize=8, radius=0.08, alpha=0.75, text_color="#333") + _box(ax, inp_cx, os_y_md, inp_w, inp_h, C_INPUT, "Clinical\nData", + fontsize=8, radius=0.08, alpha=0.75, text_color="#333") + _arrow(ax, inp_cx + inp_w / 2, os_y_img, x_left, os_y_img, lw=1.2, style="-|>") + _arrow(ax, inp_cx + inp_w / 2, os_y_md, x_left, os_y_md, lw=1.2, style="-|>") + + # Average node + avg_x = max(br_od_x, br_os_x) + 1.20 + avg_y = (cy_od + cy_os) / 2 + avg_size = 0.90 + + _arrow(ax, br_od_x, cy_od, avg_x - avg_size / 2, avg_y, lw=1.4, style="-|>") + _arrow(ax, br_os_x, cy_os, avg_x - avg_size / 2, avg_y, lw=1.4, style="-|>") + _box( + ax, + avg_x, + avg_y, + avg_size, + avg_size, + C_HEAD, + "Average", + fontsize=10, + bold=True, + radius=0.10, + ) + + # Output + out_x = avg_x + avg_size / 2 + 0.50 + _arrow(ax, avg_x + avg_size / 2, avg_y, out_x, avg_y, lw=1.5, style="-|>") + _draw_output(ax, out_x, avg_y) + + # Side brackets — white text on badge + _bracket( + ax, + x=br_od_x + 0.10, + y0=od_y_md - 0.45, + y1=od_y_img + 0.45, + text="OD HyperTower", + fontsize=8.5, + pad=0.20, + color="#555", + badge_color="#555", + ) + _bracket( + ax, + x=br_os_x + 0.10, + y0=os_y_md - 0.45, + y1=os_y_img + 0.45, + text="OS HyperTower", + fontsize=8.5, + pad=0.20, + color="#555", + badge_color="#555", + ) + + path = out_dir / "architecture_ensemble.png" + fig.savefig(path, dpi=180, bbox_inches="tight") + plt.close(fig) + print(f" Saved: {path}") + + +# ── Figure 4: Fused Head ────────────────────────────────────────────────────── + + +def make_fused_head(out_dir: Path): + W, H = 10.5, 7.5 + fig, ax = plt.subplots(figsize=(W, H)) + _setup(fig, ax, W, H, "Fused Head Bilateral HyperTower") + + x_left = 3.0 + inp_cx = 1.85 + inp_w = 0.90 + inp_h = 0.55 + + # OD (top) — same layout as ensemble + od_y_img, od_y_md = 5.90, 4.60 + br_od_x, cy_od = _draw_compact_ht( + ax, x_left=x_left, y_img=od_y_img, y_md=od_y_md, eye_label="OD" + ) + _text(ax, 0.45, (od_y_img + od_y_md) / 2, "OD\n(Right Eye)", + fontsize=9, color="#444", bold=True) + _box(ax, inp_cx, od_y_img, inp_w, inp_h, C_INPUT, "Fundus\nImage", + fontsize=8, radius=0.08, alpha=0.75, text_color="#333") + _box(ax, inp_cx, od_y_md, inp_w, inp_h, C_INPUT, "Clinical\nData", + fontsize=8, radius=0.08, alpha=0.75, text_color="#333") + _arrow(ax, inp_cx + inp_w / 2, od_y_img, x_left, od_y_img, lw=1.2, style="-|>") + _arrow(ax, inp_cx + inp_w / 2, od_y_md, x_left, od_y_md, lw=1.2, style="-|>") + + # OS (bottom) + os_y_img, os_y_md = 2.80, 1.50 + br_os_x, cy_os = _draw_compact_ht( + ax, x_left=x_left, y_img=os_y_img, y_md=os_y_md, eye_label="OS" + ) + _text(ax, 0.45, (os_y_img + os_y_md) / 2, "OS\n(Left Eye)", + fontsize=9, color="#444", bold=True) + _box(ax, inp_cx, os_y_img, inp_w, inp_h, C_INPUT, "Fundus\nImage", + fontsize=8, radius=0.08, alpha=0.75, text_color="#333") + _box(ax, inp_cx, os_y_md, inp_w, inp_h, C_INPUT, "Clinical\nData", + fontsize=8, radius=0.08, alpha=0.75, text_color="#333") + _arrow(ax, inp_cx + inp_w / 2, os_y_img, x_left, os_y_img, lw=1.2, style="-|>") + _arrow(ax, inp_cx + inp_w / 2, os_y_md, x_left, os_y_md, lw=1.2, style="-|>") + + _bracket( + ax, + x=br_od_x + -0.17, + y0=od_y_md - 0.45, + y1=od_y_img + 0.45, + text="OD HyperTower", + fontsize=8.5, + pad=0.20, + color="#555", + badge_color="#555", + ) + _bracket( + ax, + x=br_os_x + -0.17, + y0=os_y_md - 0.45, + y1=os_y_img + 0.45, + text="OS HyperTower", + fontsize=8.5, + pad=0.20, + color="#555", + badge_color="#555", + ) + + # Fused Head box with logit MLP detail + avg_y = (cy_od + cy_os) / 2 + head_x = max(br_od_x, br_os_x) + 2.20 + head_w = 1.80 + head_h = 1.20 + + _arrow(ax, br_od_x + 0.05, cy_od, head_x - head_w / 2, avg_y, lw=1.4, style="-|>") + _arrow(ax, br_os_x + 0.05, cy_os, head_x - head_w / 2, avg_y, lw=1.4, style="-|>") + _box( + ax, + head_x, + avg_y, + head_w, + head_h, + C_HEAD, + "Fused Head\ncat(l_OD, l_OS)\n→ FC(64) → logits", + fontsize=8.5, + bold=False, + radius=0.10, + ) + + # Output nodes + softmax + out_x = head_x + head_w / 2 + 0.50 + _arrow(ax, head_x + head_w / 2, avg_y, out_x, avg_y, lw=1.5, style="-|>") + _draw_output(ax, out_x, avg_y) + + path = out_dir / "architecture_fused_head.png" + fig.savefig(path, dpi=180, bbox_inches="tight") + plt.close(fig) + print(f" Saved: {path}") + + +# ── Main ────────────────────────────────────────────────────────────────────── + + +def main(): + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument( + "--out", + type=Path, + default=Path(__file__).resolve().parents[3] / "v3" / "figures", + help="Output directory", + ) + args = ap.parse_args() + args.out.mkdir(parents=True, exist_ok=True) + + print("Generating architecture diagrams...") + make_single_tower(args.out) + make_hypertower(args.out) + make_ensemble(args.out) + make_fused_head(args.out) + print("Done.") + + +if __name__ == "__main__": + main() diff --git a/v3/scripts/output_analysis/plot_cnn_backbone_comparison.py b/v3/scripts/output_analysis/plot_cnn_backbone_comparison.py new file mode 100644 index 0000000..acdb3b4 --- /dev/null +++ b/v3/scripts/output_analysis/plot_cnn_backbone_comparison.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python +""" +Bar plot comparing CNN standalone vs HyperTower image_only AUC per backbone, +with PAPILA paper reference lines. + +Usage: + python -m v3.scripts.output_analysis.plot_cnn_backbone_comparison \ + --results-dir v3/results/phase1 \ + --output v3/results/phase1/cnn_backbone_comparison.png +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from sklearn.metrics import roc_auc_score + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + + +BACKBONES = ["densenet121", "vgg16", "mobilenet_v2", "inception_v3", "resnet50"] +BACKBONE_LABELS = { + "densenet121": "DenseNet121", + "vgg16": "VGG16", + "mobilenet_v2": "MobileNetV2", + "inception_v3": "Inception V3", + "resnet50": "ResNet50", +} + +# Per-backbone paper AUCs (binary, Test #2, PAPILA 2022) +PAPER_AUC = { + "densenet121": 0.80, + "vgg16": 0.84, + "mobilenet_v2": 0.75, + "inception_v3": 0.78, + "resnet50": 0.78, +} +PAPER_STD = { + "densenet121": 0.05, + "vgg16": 0.02, + "mobilenet_v2": 0.06, + "inception_v3": 0.08, + "resnet50": 0.07, +} + +COLOURS = { + "cnn": "#4878CF", + "ht": "#D65F5F", + "paper_ref": "black", +} + + +def load_cnn_fold_aucs(results_dir: Path, backbone: str) -> list[float]: + fpath = results_dir / f"cnn_{backbone}" / "fold_metrics.csv" + if not fpath.exists(): + print(f" WARNING: missing {fpath}") + return [] + df = pd.read_csv(fpath) + return df["auc"].tolist() + + +def load_ht_fold_aucs(results_dir: Path, backbone: str, n_folds: int = 5) -> list[float]: + aucs = [] + for fold in range(n_folds): + fold_dir = results_dir / "imageonly_ht" / backbone / "binary" / "single" / f"fold{fold}" + y_path = fold_dir / "test_y_true.npy" + p_path = fold_dir / "test_probs_fused.npy" + if not (y_path.exists() and p_path.exists()): + print(f" WARNING: missing predictions for {backbone} fold{fold}") + continue + y = np.load(y_path) + pr = np.load(p_path) + if len(np.unique(y)) < 2: + print(f" WARNING: single-class test set for {backbone} fold{fold}, skipping") + continue + aucs.append(float(roc_auc_score(y, pr[:, 1]))) + return aucs + + +def plot(cnn_data: dict, ht_data: dict, output: Path): + n = len(BACKBONES) + x = np.arange(n) + group_width = 0.7 + bar_w = group_width / 2 * 0.88 + offsets = [-group_width / 4, group_width / 4] + + fig, ax = plt.subplots(figsize=(10, 5.5)) + + for bi, backbone in enumerate(BACKBONES): + for si, (tag, data, colour) in enumerate([ + ("CNN standalone", cnn_data, COLOURS["cnn"]), + ("HyperTower (image only)", ht_data, COLOURS["ht"]), + ]): + aucs = data.get(backbone, []) + if not aucs: + continue + xpos = bi + offsets[si] + mean, std = np.mean(aucs), np.std(aucs) + ax.bar( + xpos, mean, width=bar_w, + color=colour, alpha=0.80, + label=tag if bi == 0 else "_nolegend_", + ) + ax.errorbar( + xpos, mean, yerr=std, + fmt="none", color="black", capsize=4, linewidth=1.2, + ) + + # Paper reference line spanning this backbone's group + paper_val = PAPER_AUC.get(backbone) + if paper_val is not None: + lw = group_width / 2 + bar_w / 2 + label = "PAPILA paper" if bi == 0 else "_nolegend_" + ax.hlines( + paper_val, + bi - group_width / 2, bi + group_width / 2, + colors=COLOURS["paper_ref"], linestyles=":", linewidths=1.8, + label=label, + ) + + ax.set_xticks(x) + ax.set_xticklabels([BACKBONE_LABELS[b] for b in BACKBONES], fontsize=11) + ax.set_ylabel("AUC (ROC)", fontsize=11) + ax.set_title("Phase 1: CNN backbone AUC — standalone vs HyperTower (image only)", fontsize=12) + ax.set_ylim(0.45, 1.02) + ax.axhline(0.5, color="grey", linestyle="--", linewidth=0.8, alpha=0.4) + ax.grid(axis="y", alpha=0.3, linestyle="--") + ax.legend(loc="lower right", fontsize=10, framealpha=0.9) + fig.tight_layout() + + output.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(output, dpi=180) + plt.close(fig) + print(f"Saved: {output}") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--results-dir", default="v3/results/phase1") + ap.add_argument("--output", default=None) + args = ap.parse_args() + + results_dir = Path(args.results_dir) + output = Path(args.output) if args.output else results_dir / "cnn_backbone_comparison.png" + + cnn_data = {b: load_cnn_fold_aucs(results_dir, b) for b in BACKBONES} + ht_data = {b: load_ht_fold_aucs(results_dir, b) for b in BACKBONES} + + plot(cnn_data, ht_data, output) + + # Summary table + print(f"\n{'Backbone':<16} {'CNN standalone':>18} {'HT image_only':>18} {'Paper':>12}") + print("-" * 72) + for b in BACKBONES: + cnn_aucs = cnn_data[b] + ht_aucs = ht_data[b] + cnn_str = f"{np.mean(cnn_aucs):.3f} ± {np.std(cnn_aucs):.3f}" if cnn_aucs else "—" + ht_str = f"{np.mean(ht_aucs):.3f} ± {np.std(ht_aucs):.3f}" if ht_aucs else "—" + p_str = f"{PAPER_AUC[b]:.2f} ± {PAPER_STD[b]:.2f}" + print(f"{BACKBONE_LABELS[b]:<16} {cnn_str:>18} {ht_str:>18} {p_str:>12}") + + +if __name__ == "__main__": + main() diff --git a/v3/scripts/output_analysis/plot_phase3_modality_ablation.py b/v3/scripts/output_analysis/plot_phase3_modality_ablation.py new file mode 100644 index 0000000..994218d --- /dev/null +++ b/v3/scripts/output_analysis/plot_phase3_modality_ablation.py @@ -0,0 +1,119 @@ +""" +Phase 3 modality ablation — Image-only vs Clinical-only vs HyperTower (fused). + +Pools all rep×fold predictions from the phase3/baseline run and plots +per-fold AUC for each modality as a box plot with jittered points. + +Output: v3/figures/phase3_modality_ablation.png + +Usage: + python -m v3.scripts.output_analysis.plot_phase3_modality_ablation +""" +from __future__ import annotations + +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from sklearn.metrics import roc_auc_score + +REPO_ROOT = Path(__file__).resolve().parents[3] +RESULTS_DIR = REPO_ROOT / "v3" / "results" / "phase3" / "baseline" +FIGURES_DIR = REPO_ROOT / "v3" / "figures" +OUT_PNG = FIGURES_DIR / "phase3_modality_ablation.png" + +C_BASELINE = "#dd8452" +C_OTHER = "#4c72b0" +C_MEDIAN = "#c44e52" +FSIZE = 10 + +MODALITIES = [ + ("prob_img_c1", "Image only", C_OTHER), + ("prob_md_c1", "Clinical only", C_OTHER), + ("prob_fused_c1", "HyperTower\n(fused)", C_BASELINE), +] + + +def load_fold_aucs() -> dict[str, list[float]]: + aucs: dict[str, list[float]] = {col: [] for col, _, _ in MODALITIES} + + for rep_dir in sorted(RESULTS_DIR.glob("rep*")): + fold_root = rep_dir / "binary" / "single" + if not fold_root.exists(): + continue + for fold_dir in sorted(fold_root.glob("fold[0-9]")): + csv = fold_dir / "predictions_test.csv" + if not csv.exists(): + continue + df = pd.read_csv(csv) + if df["y_true"].nunique() < 2: + continue + for col, _, _ in MODALITIES: + if col in df.columns: + try: + aucs[col].append(roc_auc_score(df["y_true"], df[col])) + except Exception: + pass + + return aucs + + +def main(): + print("Loading fold AUCs ...") + aucs = load_fold_aucs() + + n_folds = len(next(iter(aucs.values()))) + print(f" {n_folds} folds found") + for col, label, _ in MODALITIES: + vals = aucs[col] + print(f" {label.replace(chr(10), ' '):<30} " + f"mean={np.mean(vals):.4f} std={np.std(vals):.4f} n={len(vals)}") + + # ── Plot ────────────────────────────────────────────────────────────────── + fig, ax = plt.subplots(figsize=(6, 4.5)) + + data_list = [np.array(aucs[col]) for col, _, _ in MODALITIES] + colors = [color for _, _, color in MODALITIES] + labels = [lbl for _, lbl, _ in MODALITIES] + x = np.arange(len(MODALITIES)) + + bp = ax.boxplot( + data_list, + vert=True, + patch_artist=True, + positions=x, + widths=0.3, + showfliers=True, + flierprops=dict(marker="o", markersize=3, alpha=0.5), + medianprops=dict(color=C_MEDIAN, linewidth=2), + ) + for patch, color in zip(bp["boxes"], colors): + patch.set_facecolor(color) + patch.set_alpha(0.8) + + ax.set_xlim(-0.5, len(MODALITIES) - 0.5) + tick_labels = [ + f"{lbl}\nAUC={np.mean(np.array(aucs[col])):.3f}" + for col, lbl, _ in MODALITIES + ] + ax.set_xticks(x) + ax.set_xticklabels(tick_labels, fontsize=FSIZE) + ax.set_ylabel("AUC (ROC)", fontsize=FSIZE + 1) + fig.suptitle( + f"Phase 3 — Modality Ablation: Image / Clinical / Fused ({n_folds} folds)", + fontsize=FSIZE + 3, fontweight="bold", + ) + ax.grid(axis="y", alpha=0.3) + + fig.tight_layout() + FIGURES_DIR.mkdir(parents=True, exist_ok=True) + fig.savefig(OUT_PNG, dpi=180, bbox_inches="tight") + plt.close(fig) + print(f"Saved: {OUT_PNG}") + + +if __name__ == "__main__": + main()