pre-refactor 041426

.gitignore

@@ -13,3 +13,5 @@ models/v2/refuge/
 .archive/
 scripts/deprecated/
 v3/results/*
+scripts/utility/backup_mirror_with_archive.sh
+v3/distributed/logs/*

scripts/utility/backup_mirror_with_archive.sh

@@ -44,7 +44,42 @@ cmd=(
   --delete
   --backup
   --backup-dir="$archive_dir"
+  # Directories with no backup value
+  --exclude=".git/"
+  --exclude=".claude/"
   --exclude=".archive/"
+  # Large datasets stored elsewhere
+  --exclude="refuge/"
+  --exclude="Refuge/"
+  --exclude="REFUGE/"
+  --exclude="papila/"
+  --exclude="Papila/"
+  --exclude="PAPILA/"
+  # Python / general caches
+  --exclude="__pycache__/"
+  --exclude=".mypy_cache/"
+  --exclude=".ruff_cache/"
+  --exclude=".pytest_cache/"
+  --exclude=".cache/"
+  --exclude="*.pyc"
+  --exclude="*.pyo"
+  # Virtual environments
+  --exclude=".venv/"
+  --exclude="venv/"
+  --exclude="env/"
+  # Node
+  --exclude="node_modules/"
+  # Build / dist artifacts
+  --exclude="*.egg-info/"
+  --exclude="dist/"
+  --exclude="build/"
+  --exclude=".eggs/"
+  # IDE / editor metadata
+  --exclude=".idea/"
+  --exclude=".vscode/"
+  # OS metadata
+  --exclude=".DS_Store"
+  --exclude="Thumbs.db"
   "${SOURCE%/}/"
   "${DEST%/}/"
 )

setup_env.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# ============================================================
+# HyperTower environment setup script.
+#
+# Installs all required packages for the v3 pipeline.
+# PyTorch is installed CPU-only by default.
+#
+# To upgrade to GPU after running this script:
+#   NVIDIA (CUDA 12.8):
+#     pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128
+#   AMD (ROCm 6.2):
+#     pip install torch torchvision --index-url https://download.pytorch.org/whl/rocm6.2
+#
+# Usage:
+#   bash setup_env.sh
+# ============================================================
+
+set -e
+
+echo "=== HyperTower environment setup ==="
+echo "Installing CPU-only PyTorch (upgrade separately for GPU)"
+echo ""
+
+# ── PyTorch (CPU) ─────────────────────────────────────────────
+pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
+
+# ── Core data science ─────────────────────────────────────────
+pip install \
+    numpy \
+    pandas \
+    scikit-learn \
+    scipy
+
+# ── Image processing ──────────────────────────────────────────
+pip install \
+    Pillow \
+    scikit-image
+
+# ── Visualisation ─────────────────────────────────────────────
+pip install \
+    matplotlib
+
+# ── Distributed job system ────────────────────────────────────
+pip install \
+    fastapi \
+    "uvicorn[standard]" \
+    requests \
+    pydantic
+
+# ── Statistics ────────────────────────────────────────────────
+pip install \
+    statsmodels
+
+# ── Utilities ─────────────────────────────────────────────────
+pip install \
+    tqdm \
+    openpyxl
+
+echo ""
+echo "=== Setup complete ==="
+echo ""
+echo "To enable GPU support:"
+echo "  NVIDIA: pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128"
+echo "  AMD:    pip install torch torchvision --index-url https://download.pytorch.org/whl/rocm6.2"

v3/classes/bridges.py

@@ -14,6 +14,7 @@ class Bridge(nn.Module):
         num_classes,
         fusion_dim=256,
         mode="fused",
+        dropout: float = 0.5,
         use_se: bool = True,
         se_reduction: int = 16,
         se_pre_norm: bool = True,
@@ -37,7 +38,7 @@ class Bridge(nn.Module):
         # heads
         self.classifier_fused = nn.Sequential(
             nn.ReLU(),
-            nn.Dropout(0.5),
+            nn.Dropout(dropout),
             nn.Linear(fusion_dim, num_classes),
         )
         self.classifier_img = nn.Linear(img_dim, num_classes)
@@ -54,26 +55,28 @@ class Bridge(nn.Module):
             return self.se_log.get(reset=reset)
         return None
 
+    def _compute_fused(self, img_feats, md_feats):
+        """Return z_fused embedding (before classifier_fused). Used by encode() and forward()."""
+        hi = self.ln_img(self.W_img(img_feats))
+        hm = self.ln_md(self.W_md(md_feats))
+        fused = hi * hm
+        if self.se is not None:
+            fused, gates = self.se(fused)
+            if self.se_log.enabled:
+                self.se_log.accumulate(gates)
+        return fused
+
+    def encode(self, img_feats, md_feats) -> torch.Tensor:
+        """Return z_fused embedding without applying the classifier head."""
+        assert self.mode == "fused", "encode() only valid in fused mode"
+        return self._compute_fused(img_feats, md_feats)
+
     def forward(self, img_feats, md_feats):
         out_img = None if self.mode == "clinical_only" else self.classifier_img(img_feats)
         out_md = None if self.mode == "image_only" else self.classifier_cd(md_feats)
 
         if self.mode == "fused":
-            hi = self.ln_img(self.W_img(img_feats))  # image features
+            fused = self._compute_fused(img_feats, md_feats)
-            hm = self.ln_md(self.W_md(md_feats))  # clinical data features
-            fused = hi * hm  # elementwise product
-            # apply SE gates
-            if self.se is not None:
-                fused, gates = self.se(fused)
-                if self.se_log.enabled:
-                    self.se_log.accumulate(gates)
-
-            if self.se is not None and self.training and self.se_log.enabled:
-                if not hasattr(self, "_dbg_seen"):
-                    self._dbg_seen = 0
-                if self._dbg_seen < 3:  # print only a few times
-                    print("[SE] gate mean this batch:", gates.mean().item())
-                    self._dbg_seen += 1
             out_f = self.classifier_fused(fused)
             return out_f, out_img, out_md
         # if ablation modes:

v3/classes/croppers.py

@@ -189,6 +189,31 @@ class UNetImageCropper:
             return None
         return np.asarray(features, dtype=np.float32)
 
+    def precompute_geometry(self, image_paths) -> None:
+        """Pre-compute geometry features for all image_paths into an in-memory cache.
+        Safe to call in the main process; geometry_for_image() can then be called
+        from DataLoader workers without touching CUDA.
+        """
+        self._geometry_cache: Dict[str, Optional[np.ndarray]] = {}
+        paths = list(image_paths)
+        print(f"[UNetImageCropper] pre-computing geometry for {len(paths)} images...", flush=True)
+        for img_path in paths:
+            key = str(Path(img_path).resolve())
+            try:
+                img = Image.open(img_path).convert("RGB")
+                self._geometry_cache[key] = self.geometry_features(img, img_path)
+            except Exception:
+                self._geometry_cache[key] = None
+        n_ok = sum(1 for v in self._geometry_cache.values() if v is not None)
+        print(f"[UNetImageCropper] {n_ok}/{len(paths)} geometry vectors computed", flush=True)
+
+    def geometry_for_image(self, image_path) -> Optional[np.ndarray]:
+        """Return pre-computed geometry vector for image_path (call precompute_geometry first)."""
+        cache = getattr(self, "_geometry_cache", None)
+        if cache is None:
+            raise RuntimeError("Call precompute_geometry() before geometry_for_image()")
+        return cache.get(str(Path(image_path).resolve()))
+
 
 class ManifestImageCropper:
     def __init__(
@@ -371,6 +396,35 @@ class ManifestImageCropper:
             return None
         return np.asarray(features, dtype=np.float32)
 
+    def precompute_geometry(self, image_paths) -> None:
+        """Pre-compute geometry features for all image_paths into an in-memory cache.
+        Safe to call in the main process; geometry_for_image() can then be called
+        without re-opening images or re-loading annotations.
+        """
+        self._geometry_cache: Dict[str, Optional[np.ndarray]] = {}
+        paths = list(image_paths)
+        print(f"[ManifestImageCropper] pre-computing geometry for {len(paths)} images...", flush=True)
+        for img_path in paths:
+            key = str(Path(img_path).resolve())
+            entry = self.entries.get(key)
+            if entry is None:
+                self._geometry_cache[key] = None
+                continue
+            try:
+                img = Image.open(img_path).convert("RGB")
+                self._geometry_cache[key] = self.geometry_features(img, img_path)
+            except Exception:
+                self._geometry_cache[key] = None
+        n_ok = sum(1 for v in self._geometry_cache.values() if v is not None)
+        print(f"[ManifestImageCropper] {n_ok}/{len(paths)} geometry vectors computed", flush=True)
+
+    def geometry_for_image(self, image_path) -> Optional[np.ndarray]:
+        """Return pre-computed geometry vector for image_path (call precompute_geometry first)."""
+        cache = getattr(self, "_geometry_cache", None)
+        if cache is None:
+            raise RuntimeError("Call precompute_geometry() before geometry_for_image()")
+        return cache.get(str(Path(image_path).resolve()))
+
 
 # ---------------------------------------------------------------------------
 # Factory

v3/classes/loader_factory.py

@@ -211,6 +211,7 @@ def make_loader(
     shuffle: bool,
     num_workers: int,
     sampler: Optional[WeightedRandomSampler] = None,
+    persistent_workers: bool = False,
 ) -> DataLoader:
     ds = SlotDataset(
         samples,
@@ -226,6 +227,7 @@ def make_loader(
         sampler=sampler,
         num_workers=num_workers,
         collate_fn=slot_collate,
+        persistent_workers=(persistent_workers and num_workers > 0),
     )
 
 

v3/classes/models.py

@@ -11,7 +11,7 @@ from torch import nn
 from torch.utils.data import DataLoader
 
 from v3.classes.bridges import Bridge
-from v3.classes.towers import ImageTower, ClinicalTower
+from v3.classes.towers import ImageTower, ClinicalTower, SiameseImageTower
 
 
 # ---------------------------------------------------------------------------
@@ -35,18 +35,24 @@ class SingleEyeHT(nn.Module):
         cd_hidden_dim: int = 128,
         fusion_dim: int = 256,
         bridge_mode: str = "fused",
+        bridge_dropout: float = 0.5,
+        cd_dropout: float = 0.1,
+        se_img_tower: bool = False,
+        se_cd_tower: bool = False,
+        se_bridge: bool = False,
     ):
         super().__init__()
         self.img_tower = ImageTower(
             backbone=backbone,
             freeze_ratio=freeze_ratio,
             augment=augment,
-            use_se=False,
+            use_se=se_img_tower,
         )
         self.cd_tower = ClinicalTower(
             clinical_data=clinical_data,
             hidden_dim=cd_hidden_dim,
-            use_se=False,
+            dropout=cd_dropout,
+            use_se=se_cd_tower,
         )
         self.bridge = Bridge(
             img_dim=self.img_tower.out_dim,
@@ -54,13 +60,20 @@ class SingleEyeHT(nn.Module):
             num_classes=num_classes,
             fusion_dim=fusion_dim,
             mode=bridge_mode,
-            use_se=False,
+            dropout=bridge_dropout,
+            use_se=se_bridge,
         )
 
     @property
     def transform(self):
         return self.img_tower.transform
 
+    def encode(self, x: torch.Tensor, meta: torch.Tensor) -> torch.Tensor:
+        """Return z_fused embedding (fusion_dim) without applying the classifier head."""
+        img_feats = self.img_tower(x)
+        md_feats  = self.cd_tower(meta)
+        return self.bridge.encode(img_feats, md_feats)
+
     def forward(self, x: torch.Tensor, meta: torch.Tensor) -> torch.Tensor:
         img_feats = None if self.bridge.mode == "clinical_only" else self.img_tower(x)
         md_feats  = None if self.bridge.mode == "image_only"    else self.cd_tower(meta)
@@ -158,6 +171,59 @@ class BilateralHT(nn.Module):
         return out_f
 
 
+class SiameseHT(nn.Module):
+    """
+    Bilateral model using a shared-weight SiameseImageTower (mean+delta).
+
+    Both eyes pass through the same backbone; features are combined as
+    cat([mean(f_od, f_os), f_od - f_os]) giving the model both a shared
+    bilateral representation and an asymmetry signal.
+
+    Uses the same bilateral loader and training loop as BilateralHT.
+    The bridge operates in image_only mode (no clinical data in phase 4).
+    """
+
+    def __init__(
+        self,
+        *,
+        backbone: str,
+        freeze_ratio: float,
+        augment: bool,
+        num_classes: int,
+        fusion_dim: int = 256,
+        bridge_mode: str = "image_only",
+    ):
+        super().__init__()
+        self.img_tower = SiameseImageTower(
+            backbone=backbone,
+            freeze_ratio=freeze_ratio,
+            augment=augment,
+            use_se=False,
+        )
+        img_dim = self.img_tower.out_dim  # 2 * backbone_out_dim
+        self.bridge = Bridge(
+            img_dim=img_dim,
+            meta_dim=1,            # dummy — not used in image_only mode
+            num_classes=num_classes,
+            fusion_dim=fusion_dim,
+            mode="image_only",
+            use_se=False,
+        )
+        self.aux_img = nn.Linear(img_dim, num_classes)
+
+    @property
+    def transform(self):
+        return self.img_tower.transform
+
+    def encode(self, x_od: torch.Tensor, x_os: torch.Tensor) -> torch.Tensor:
+        return self.img_tower(x_od, x_os)
+
+    def forward(self, x_od: torch.Tensor, x_os: torch.Tensor) -> torch.Tensor:
+        feats = self.encode(x_od, x_os)
+        out_f, _, _ = self.bridge(feats, None)
+        return out_f
+
+
 class FusedEnsembleHT(nn.Module):
     """
     SingleEyeHT base with a per-eye attention scorer for bilateral fusion.
@@ -190,6 +256,10 @@ class FusedEnsembleHT(nn.Module):
         # Learns the GC-direction in logit space from bilateral labels.
         self.eye_scorer = nn.Linear(num_classes, 1, bias=True)
 
+    @property
+    def head(self) -> nn.Module:
+        return self.eye_scorer
+
     def forward(
         self,
         x_od:    torch.Tensor,
@@ -205,6 +275,78 @@ class FusedEnsembleHT(nn.Module):
         return alpha[:, 0:1] * logit_od + alpha[:, 1:2] * logit_os            # [B, C]
 
 
+class LogitMLPEnsembleHT(nn.Module):
+    """
+    MLP head trained on concatenated per-eye logits.
+
+    Both eyes pass through the frozen base independently, producing per-eye
+    logit vectors.  These are concatenated and fed through a small MLP:
+
+        cat([logit_od, logit_os])  [B, 2C]
+          → Linear(2C, hidden) → ReLU → Dropout → Linear(hidden, C)
+
+    Permutation-variant by design: the model can learn left/right asymmetries
+    directly from the concatenated pair, at the cost of needing a consistent
+    OD-first input ordering.
+    """
+
+    def __init__(self, base: SingleEyeHT, num_classes: int, hidden: int = 64):
+        super().__init__()
+        self.base = base
+        self.head = nn.Sequential(
+            nn.Linear(2 * num_classes, hidden),
+            nn.ReLU(),
+            nn.Dropout(0.3),
+            nn.Linear(hidden, num_classes),
+        )
+
+    def forward(
+        self,
+        x_od: torch.Tensor, meta_od: torch.Tensor,
+        x_os: torch.Tensor, meta_os: torch.Tensor,
+    ) -> torch.Tensor:
+        logit_od = self.base(x_od, meta_od)
+        logit_os = self.base(x_os, meta_os)
+        return self.head(torch.cat([logit_od, logit_os], dim=1))
+
+
+class EmbeddingMLPEnsembleHT(nn.Module):
+    """
+    MLP head trained on concatenated per-eye z_fused embeddings.
+
+    Both eyes pass through the frozen base independently, and their bridge
+    embeddings (pre-classifier, shape [B, fusion_dim]) are concatenated and
+    fed through an MLP:
+
+        cat([z_od, z_os])  [B, 2 * fusion_dim]
+          → Linear(2*fusion_dim, hidden) → ReLU → Dropout → Linear(hidden, C)
+
+    Richer than logit-level: the head sees pre-softmax feature vectors rather
+    than the compressed C-dimensional output, giving it more signal to work
+    with when fusion_dim >> C.
+    """
+
+    def __init__(self, base: SingleEyeHT, num_classes: int, hidden: int = 256):
+        super().__init__()
+        self.base = base
+        fusion_dim = base.bridge.W_img.out_features
+        self.head = nn.Sequential(
+            nn.Linear(2 * fusion_dim, hidden),
+            nn.ReLU(),
+            nn.Dropout(0.3),
+            nn.Linear(hidden, num_classes),
+        )
+
+    def forward(
+        self,
+        x_od: torch.Tensor, meta_od: torch.Tensor,
+        x_os: torch.Tensor, meta_os: torch.Tensor,
+    ) -> torch.Tensor:
+        z_od = self.base.encode(x_od, meta_od)
+        z_os = self.base.encode(x_os, meta_os)
+        return self.head(torch.cat([z_od, z_os], dim=1))
+
+
 # ---------------------------------------------------------------------------
 # Phase control
 # ---------------------------------------------------------------------------
@@ -429,15 +571,89 @@ def train_bilateral_epoch(
     )
 
 
+def train_siamese_epoch(
+    model: SiameseHT,
+    loader: DataLoader,
+    opt,
+    device: torch.device,
+    *,
+    bcd_prob: float = 0.5,
+    tower_loss_mode: str = "bcd",
+) -> tuple[float, float]:
+    """Train one epoch of SiameseHT on bilateral (patient-level) samples."""
+    model.train()
+    total_loss = total_correct = total_n = 0
+    for batch in loader:
+        x1 = batch.get("image_1")
+        x2 = batch.get("image_2")
+        y  = batch.get("label_1")
+        if not (torch.is_tensor(x1) and torch.is_tensor(x2)):
+            continue
+        x1 = x1.to(device); x2 = x2.to(device)
+        y  = _to_label_tensor(y, device)
+        feats = model.encode(x1, x2)
+
+        if tower_loss_mode == "all":
+            logits_i = model.aux_img(feats)
+            out_f, _, _ = model.bridge(feats, None)
+            loss = F.cross_entropy(out_f, y) + F.cross_entropy(logits_i, y)
+            logits = out_f
+        elif random() < bcd_prob:
+            logits = model.aux_img(feats)
+            loss = F.cross_entropy(logits, y)
+        else:
+            logits, _, _ = model.bridge(feats, None)
+            loss = F.cross_entropy(logits, y)
+
+        opt.zero_grad()
+        loss.backward()
+        opt.step()
+        bs = y.shape[0]
+        total_loss    += float(loss.item()) * bs
+        total_correct += int((logits.argmax(1) == y).sum())
+        total_n       += bs
+    return (
+        total_loss / total_n if total_n else float("nan"),
+        total_correct / total_n if total_n else float("nan"),
+    )
+
+
+def collect_probs_siamese(
+    model: SiameseHT,
+    loader: DataLoader,
+    device: torch.device,
+) -> tuple[np.ndarray, np.ndarray]:
+    """Patient-level probs from a bilateral loader using SiameseHT."""
+    model.eval()
+    y_c, p_c = [], []
+    with torch.no_grad():
+        for batch in loader:
+            x1 = batch.get("image_1")
+            x2 = batch.get("image_2")
+            y  = batch.get("label_1")
+            if not (torch.is_tensor(x1) and torch.is_tensor(x2)):
+                continue
+            x1 = x1.to(device); x2 = x2.to(device)
+            feats = model.encode(x1, x2)
+            logits, _, _ = model.bridge(feats, None)
+            probs = torch.softmax(logits, dim=1)
+            y_c.append(np.array(y) if not torch.is_tensor(y) else y.cpu().numpy())
+            p_c.append(probs.cpu().numpy())
+    return (
+        np.concatenate(y_c, axis=0),
+        np.concatenate(p_c, axis=0),
+    )
+
+
 def train_fusion_epoch(
-    model: FusedEnsembleHT,
+    model,  # FusedEnsembleHT | LogitMLPEnsembleHT | EmbeddingMLPEnsembleHT
     loader: DataLoader,
     opt,
     device: torch.device,
 ) -> tuple[float, float]:
     """Train only the fusion head; the base SingleEyeHT is frozen in eval mode."""
     model.base.eval()
-    model.eye_scorer.train()
+    model.head.train()
     total_loss = total_correct = total_n = 0
     for batch in loader:
         x1 = batch.get("image_1"); m1 = batch.get("matrix_1")

v3/classes/predictions.py

@@ -31,6 +31,8 @@ def head_names_for_mode(tower_mode: str, *, fused_head: bool = False) -> list[st
         return names + ["bilat_fused"] if fused_head else names
     if tower_mode == "bilateral":
         return ["fused", "img_joint", "md_joint"]
+    if tower_mode == "siamese":
+        return ["fused"]
     raise ValueError(f"Unknown tower_mode: {tower_mode!r}")
 
 

v3/classes/seg_cnn.py

@@ -0,0 +1,521 @@
+"""Segmentation-map CNN for glaucoma grading.
+
+Trains a CNN on combined disc/cup segmentation maps — pixel values
+  0 = background,  1 = disc (rim only),  2 = cup
+— instead of raw RGB fundus images, forcing the model to learn purely
+from optic nerve head geometry (CDR, rim width, cup location, etc.).
+
+Two segmentation sources are supported:
+  gt    – rasterise expert contour/mask annotations directly (pure NumPy/PIL,
+          no CUDA — safe in DataLoader worker processes)
+  unet  – run a trained UNetSegmenter on the raw fundus image
+
+Usage (import from training script):
+    from v3.classes.seg_cnn import SegMapRecord, SegMapDataset, SegCNN, seg_map_to_tensor
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+import numpy as np
+import torch
+import torch.nn as nn
+from PIL import Image, ImageDraw
+from PIL.Image import Resampling
+from torch.utils.data import Dataset
+from torchvision import models, transforms
+from tqdm import tqdm
+
+
+# ---------------------------------------------------------------------------
+# Data record
+# ---------------------------------------------------------------------------
+
+@dataclass
+class SegMapRecord:
+    """One labelled eye sample for the seg-map CNN."""
+    sample_id: str
+    image_path: Path           # original fundus image (used by unet mode)
+    annotation_disc: Path      # contour (.txt) or mask (.bmp/.png)
+    annotation_cup: Path
+    annotation_type_disc: str  # "contour" or "mask"
+    annotation_type_cup: str
+    patient_id: int            # for group-CV: keep both eyes of a patient together
+    eye: str                   # "OD" or "OS"
+    label: int                 # 0 = Normal, 1 = Glaucoma
+
+
+# ---------------------------------------------------------------------------
+# Seg-map utilities
+# ---------------------------------------------------------------------------
+
+def _combine_masks(disc_mask: np.ndarray, cup_mask: np.ndarray) -> np.ndarray:
+    """
+    Combine binary disc and cup masks into a 3-class label map.
+
+    Returns a uint8 array with values:
+      0 — background
+      1 — optic disc rim  (disc but not cup)
+      2 — optic cup
+    """
+    disc = (disc_mask > 0).astype(np.uint8)
+    cup  = (cup_mask  > 0).astype(np.uint8)
+    cup  = (cup & disc)          # structural prior: cup must be inside disc
+    seg  = disc + cup            # 0, 1 (rim), or 2 (cup)
+    return seg.astype(np.uint8)
+
+
+def crop_to_disc(seg_map: np.ndarray) -> np.ndarray:
+    """
+    Crop a seg map tightly to the disc bounding box.
+
+    The disc is anywhere seg_map > 0 (i.e. rim or cup).
+    Returns the original array unchanged if no disc is found.
+    """
+    rows = np.any(seg_map > 0, axis=1)
+    cols = np.any(seg_map > 0, axis=0)
+    if not rows.any():
+        return seg_map
+    r0, r1 = int(np.argmax(rows)), int(len(rows) - 1 - np.argmax(rows[::-1]))
+    c0, c1 = int(np.argmax(cols)), int(len(cols) - 1 - np.argmax(cols[::-1]))
+    return seg_map[r0:r1 + 1, c0:c1 + 1]
+
+
+def seg_map_to_tensor(
+    seg_map: np.ndarray,
+    channels: int,
+    target_size: int,
+) -> torch.Tensor:
+    """
+    Convert an (H, W) seg map with values {0, 1, 2} to a float tensor.
+
+    channels=1  →  (1, H, W) float in [0, 1]  (values 0/0.5/1.0)
+    channels=3  →  (3, H, W) one-hot binary channels [bg, disc_rim, cup]
+    """
+    pil = Image.fromarray(seg_map.astype(np.uint8), mode="L")
+    pil = pil.resize((target_size, target_size), Image.NEAREST)
+    seg = np.array(pil, dtype=np.uint8)
+
+    if channels == 1:
+        arr = seg.astype(np.float32) / 2.0   # {0, 0.5, 1.0}
+        return torch.from_numpy(arr).unsqueeze(0)
+
+    if channels == 3:
+        bg       = (seg == 0).astype(np.float32)
+        disc_rim = (seg == 1).astype(np.float32)
+        cup      = (seg == 2).astype(np.float32)
+        return torch.from_numpy(np.stack([bg, disc_rim, cup], axis=0))
+
+    raise ValueError(f"channels must be 1 or 3, got {channels}")
+
+
+# ---------------------------------------------------------------------------
+# GT mask loading (pure NumPy / PIL — no CUDA, safe in DataLoader workers)
+# ---------------------------------------------------------------------------
+
+def _load_contour(path: Path) -> np.ndarray:
+    """Load x,y contour pairs from a whitespace- or comma-delimited text file."""
+    for delimiter in (",", None):
+        try:
+            arr = np.loadtxt(str(path), delimiter=delimiter, comments="#", dtype=np.float32)
+            if arr.size > 0:
+                break
+        except Exception:
+            arr = np.zeros((0, 2), dtype=np.float32)
+    if arr.size == 0 or arr.ndim == 1:
+        return np.zeros((0, 2), dtype=np.float32)
+    if arr.shape[1] < 2:
+        return np.zeros((0, 2), dtype=np.float32)
+    return arr[:, :2]
+
+
+def _contour_to_mask(
+    coords: np.ndarray, image_size: Tuple[int, int], target_size: int
+) -> np.ndarray:
+    """
+    Rasterise a polygon defined by (x, y) coords into a binary mask.
+
+    image_size is the (width, height) of the original fundus image — the
+    coordinate space the contour was annotated in.  The mask is drawn at
+    that resolution then resized to target_size, matching UNetSegmenter's
+    behaviour and avoiding off-canvas clipping.
+    """
+    if coords is None or len(coords) < 3:
+        return np.zeros((target_size, target_size), dtype=np.uint8)
+    points = [tuple(map(float, pt)) for pt in coords]
+    img = Image.new("L", image_size, 0)
+    ImageDraw.Draw(img).polygon(points, outline=1, fill=1)
+    img = img.resize((target_size, target_size), Resampling.NEAREST)
+    return (np.array(img, dtype=np.uint8) > 0).astype(np.uint8)
+
+
+def _extract_masks_from_image(
+    mask_path: Path, target_size: int
+) -> Tuple[np.ndarray, np.ndarray]:
+    """
+    Extract disc and cup binary masks from a segmentation image file.
+
+    Handles both grayscale label images (e.g. REFUGE .bmp) and
+    RGB colour-coded masks.  Returns (disc_mask, cup_mask) both at
+    target_size × target_size.
+    """
+    raw = Image.open(mask_path)
+    arr = np.array(raw)
+
+    if arr.ndim == 2:
+        # Grayscale: identify background from edge statistics
+        edges = np.concatenate([arr[0], arr[-1], arr[:, 0], arr[:, -1]])
+        bg_val = int(np.argmax(np.bincount(edges.astype(np.int64).clip(0, 255), minlength=256)))
+        disc_arr = (arr != bg_val).astype(np.uint8)
+        vals = np.unique(arr)
+        non_bg = vals[vals != bg_val]
+        cup_arr: np.ndarray
+        if non_bg.size > 1:
+            cup_val = int(non_bg.min())
+            cup_arr = (arr == cup_val).astype(np.uint8)
+        else:
+            cup_arr = np.zeros_like(disc_arr, dtype=np.uint8)
+    else:
+        img_rgb = raw.convert("RGB")
+        arr = np.array(img_rgb)
+        h, w, c = arr.shape
+        edges_rgb = np.concatenate(
+            [arr[0], arr[-1], arr[:, 0], arr[:, -1]], axis=0
+        )
+        edge_colors, edge_counts = np.unique(edges_rgb.reshape(-1, c), axis=0, return_counts=True)
+        bg_color = edge_colors[int(np.argmax(edge_counts))]
+        colors, counts = np.unique(arr.reshape(-1, c), axis=0, return_counts=True)
+        not_bg = np.any(colors != bg_color.reshape(1, -1), axis=1)
+        colors, counts = colors[not_bg], counts[not_bg]
+        disc_arr = np.zeros((h, w), dtype=np.uint8)
+        cup_arr  = np.zeros((h, w), dtype=np.uint8)
+        if colors.shape[0] >= 1:
+            order = np.argsort(-counts)
+            disc_color = colors[order[0]]
+            disc_arr[np.all(arr == disc_color, axis=-1)] = 1
+            if colors.shape[0] >= 2:
+                cup_color = colors[order[1]]
+                cup_arr[np.all(arr == cup_color, axis=-1)] = 1
+
+    # Resize to target_size with nearest-neighbour to preserve binary values
+    def _resize(m: np.ndarray) -> np.ndarray:
+        pil = Image.fromarray((m > 0).astype(np.uint8) * 255)
+        pil = pil.resize((target_size, target_size), Resampling.NEAREST)
+        return (np.array(pil) > 0).astype(np.uint8)
+
+    return _resize(disc_arr), _resize(cup_arr)
+
+
+def load_gt_masks(rec: "SegMapRecord", target_size: int) -> Tuple[np.ndarray, np.ndarray]:
+    """
+    Load GT disc + cup masks for one record.
+
+    Handles annotation_type "contour" (x,y text file) and "mask" (image file).
+    Returns (disc_mask, cup_mask) as uint8 arrays of shape (target_size, target_size).
+    """
+    disc_mask: Optional[np.ndarray] = None
+    cup_mask:  Optional[np.ndarray] = None
+
+    # Get original image size so contour coordinates are drawn in the right space
+    with Image.open(rec.image_path) as _img:
+        image_size = _img.size  # (width, height)
+
+    # ---- Disc ----
+    if rec.annotation_type_disc == "mask":
+        disc_mask, cup_from_disc = _extract_masks_from_image(rec.annotation_disc, target_size)
+        if cup_from_disc.any():
+            cup_mask = cup_from_disc
+    else:  # contour
+        coords = _load_contour(rec.annotation_disc)
+        disc_mask = _contour_to_mask(coords, image_size, target_size)
+
+    # ---- Cup ----
+    if cup_mask is None:
+        if rec.annotation_type_cup == "mask":
+            _, cup_from_cup = _extract_masks_from_image(rec.annotation_cup, target_size)
+            cup_mask = cup_from_cup
+        else:  # contour
+            coords = _load_contour(rec.annotation_cup)
+            cup_mask = _contour_to_mask(coords, image_size, target_size)
+
+    if disc_mask is None:
+        disc_mask = np.zeros((target_size, target_size), dtype=np.uint8)
+    if cup_mask is None:
+        cup_mask = np.zeros((target_size, target_size), dtype=np.uint8)
+
+    # Structural prior: cup must lie within disc
+    cup_mask = (cup_mask > 0) & (disc_mask > 0)
+    return disc_mask.astype(np.uint8), cup_mask.astype(np.uint8)
+
+
+# ---------------------------------------------------------------------------
+# U-Net fine-tuning dataset
+# ---------------------------------------------------------------------------
+
+class UNetFineTuneDataset(Dataset):
+    """
+    Loads (image_tensor, mask_tensor) pairs for fine-tuning the U-Net on
+    PAPILA GT annotations.  Uses the same preprocessing as UNetSegmenter
+    so the fine-tuned weights are compatible with inference.
+    """
+
+    def __init__(
+        self,
+        records: List[SegMapRecord],
+        target_size: int = 512,
+        normalize: str = "per_image",
+    ) -> None:
+        self.records = records
+        self.target_size = target_size
+        self.normalize = normalize
+        self.to_tensor = transforms.ToTensor()
+
+    def __len__(self) -> int:
+        return len(self.records)
+
+    def _normalize(self, tensor: torch.Tensor) -> torch.Tensor:
+        if self.normalize == "per_image":
+            mean = tensor.mean(dim=(1, 2), keepdim=True)
+            std  = tensor.std(dim=(1, 2), keepdim=True).clamp(min=1e-6)
+            return (tensor - mean) / std
+        if self.normalize == "imagenet":
+            mean = torch.tensor([0.485, 0.456, 0.406]).view(-1, 1, 1)
+            std  = torch.tensor([0.229, 0.224, 0.225]).view(-1, 1, 1)
+            return (tensor - mean) / std
+        return tensor
+
+    def __getitem__(self, idx: int):
+        rec = self.records[idx]
+        image = Image.open(rec.image_path).convert("RGB")
+        image = image.resize((self.target_size, self.target_size), Resampling.BILINEAR)
+        img_tensor = self._normalize(self.to_tensor(image))
+
+        disc_mask, cup_mask = load_gt_masks(rec, self.target_size)
+        mask_tensor = torch.from_numpy(
+            np.stack([disc_mask, cup_mask], axis=0).astype(np.float32)
+        )
+        return img_tensor, mask_tensor
+
+
+# ---------------------------------------------------------------------------
+# U-Net precomputation (run once per full record list, not per fold)
+# ---------------------------------------------------------------------------
+
+def precompute_unet_seg_maps(
+    records: List["SegMapRecord"],
+    segmenter,
+    threshold: float = 0.5,
+) -> List[np.ndarray]:
+    """
+    Run the U-Net on every record and return a list of combined seg maps.
+
+    Call this once before the CV loop and pass the results to each fold's
+    SegMapDataset via precomputed_seg_maps, so the U-Net isn't re-run per fold.
+    """
+    to_tensor = transforms.ToTensor()
+    seg_maps = []
+    for rec in tqdm(records, desc="U-Net inference", unit="img", leave=False):
+        image = Image.open(rec.image_path).convert("RGB")
+        resized = segmenter.preprocess_image(image)
+        tensor = segmenter._normalize_tensor(
+            to_tensor(resized).to(segmenter.device)
+        ).unsqueeze(0)
+        with torch.no_grad():
+            logits = segmenter.model(tensor)
+        probs = torch.sigmoid(logits)[0].cpu().numpy()
+        disc = (probs[0] > threshold).astype(np.uint8)
+        cup  = (probs[1] > threshold).astype(np.uint8)
+        cup  = (cup & disc)
+        seg_maps.append(_combine_masks(disc, cup.astype(np.uint8)))
+    return seg_maps
+
+
+# ---------------------------------------------------------------------------
+# Dataset
+# ---------------------------------------------------------------------------
+
+class SegMapDataset(Dataset):
+    """
+    PyTorch Dataset that yields (seg_tensor, label) pairs.
+
+    Parameters
+    ----------
+    records      : list of SegMapRecord
+    target_size  : CNN input spatial size (images are resized to this)
+    channels     : 1 = single-channel label map;  3 = one-hot three channels
+    augment      : apply random flips + rotation (for training set)
+    unet_segmenter : if provided, use U-Net predictions instead of GT masks;
+                     must be a loaded UNetSegmenter with model weights set
+    unet_threshold : threshold for U-Net logit → binary mask
+    seg_target_size: resolution at which GT masks are rasterised (or U-Net
+                     output size).  Default 512 matches UNetSegmenter default.
+    crop_to_disc   : crop the seg map tightly to the disc bounding box before
+                     resizing to target_size (default True — eliminates the
+                     background zeros that make up most of the full image)
+    """
+
+    def __init__(
+        self,
+        records: List[SegMapRecord],
+        target_size: int = 224,
+        channels: int = 3,
+        augment: bool = False,
+        unet_segmenter=None,
+        unet_threshold: float = 0.5,
+        seg_target_size: int = 512,
+        crop_to_disc: bool = True,
+        precomputed_seg_maps: Optional[List[np.ndarray]] = None,
+    ) -> None:
+        self.records = records
+        self.target_size = target_size
+        self.channels = channels
+        self.augment = augment
+        self.seg_target_size = seg_target_size
+        self.crop_to_disc = crop_to_disc
+
+        if precomputed_seg_maps is not None:
+            self._seg_maps = precomputed_seg_maps
+        elif unet_segmenter is not None:
+            self._seg_maps = precompute_unet_seg_maps(
+                records, unet_segmenter, unet_threshold
+            )
+        else:
+            self._seg_maps = None
+
+    # ------------------------------------------------------------------
+    def __len__(self) -> int:
+        return len(self.records)
+
+    # ------------------------------------------------------------------
+    def _augment(self, seg_map: np.ndarray) -> np.ndarray:
+        """Random flips + 90° rotations (label-safe since NEAREST resize)."""
+        if np.random.rand() < 0.5:
+            seg_map = np.fliplr(seg_map)
+        if np.random.rand() < 0.5:
+            seg_map = np.flipud(seg_map)
+        k = np.random.randint(0, 4)
+        if k:
+            seg_map = np.rot90(seg_map, k=k)
+        return np.ascontiguousarray(seg_map)
+
+    # ------------------------------------------------------------------
+    def __getitem__(self, idx: int):
+        rec = self.records[idx]
+
+        if self._seg_maps is not None:
+            seg_map = self._seg_maps[idx]
+        else:
+            disc_mask, cup_mask = load_gt_masks(rec, self.seg_target_size)
+            seg_map = _combine_masks(disc_mask, cup_mask)
+
+        if self.crop_to_disc:
+            seg_map = crop_to_disc(seg_map)
+
+        if self.augment:
+            seg_map = self._augment(seg_map)
+
+        tensor = seg_map_to_tensor(seg_map, self.channels, self.target_size)
+        return tensor, rec.label
+
+
+# ---------------------------------------------------------------------------
+# Model
+# ---------------------------------------------------------------------------
+
+class SegCNN(nn.Module):
+    """
+    Pretrained CNN backbone adapted for segmentation-map input.
+
+    Parameters
+    ----------
+    num_classes : output classes (2 for binary glaucoma grading)
+    backbone    : "resnet18" | "resnet50" | "efficientnet_b0"
+    pretrained  : initialise with ImageNet weights (recommended even for
+                  non-RGB input — transfer generalises across domains)
+    in_channels : 1 (single label map) or 3 (one-hot channels)
+    dropout     : dropout rate before the final classifier head
+    """
+
+    def __init__(
+        self,
+        num_classes: int = 2,
+        backbone: str = "resnet18",
+        pretrained: bool = True,
+        in_channels: int = 3,
+        dropout: float = 0.3,
+    ) -> None:
+        super().__init__()
+
+        weights_arg = "DEFAULT" if pretrained else None
+
+        if backbone == "resnet18":
+            base = models.resnet18(weights=weights_arg)
+            feat_dim = base.fc.in_features
+            base.fc = nn.Identity()
+        elif backbone == "resnet50":
+            base = models.resnet50(weights=weights_arg)
+            feat_dim = base.fc.in_features
+            base.fc = nn.Identity()
+        elif backbone == "efficientnet_b0":
+            base = models.efficientnet_b0(weights=weights_arg)
+            feat_dim = base.classifier[1].in_features
+            base.classifier = nn.Identity()
+        else:
+            raise ValueError(f"Unknown backbone: {backbone!r}")
+
+        # Adapt first conv layer if in_channels ≠ 3
+        if in_channels != 3:
+            first_conv = self._find_first_conv(base)
+            new_conv = nn.Conv2d(
+                in_channels,
+                first_conv.out_channels,
+                kernel_size=first_conv.kernel_size,
+                stride=first_conv.stride,
+                padding=first_conv.padding,
+                bias=first_conv.bias is not None,
+            )
+            if pretrained:
+                # Average pretrained RGB weights across channel dim
+                with torch.no_grad():
+                    new_conv.weight.copy_(
+                        first_conv.weight.mean(dim=1, keepdim=True).expand_as(new_conv.weight)
+                    )
+            self._replace_first_conv(base, new_conv)
+
+        self.backbone = base
+        self.head = nn.Sequential(
+            nn.Dropout(p=dropout),
+            nn.Linear(feat_dim, num_classes),
+        )
+
+    # ------------------------------------------------------------------
+    @staticmethod
+    def _find_first_conv(module: nn.Module) -> nn.Conv2d:
+        for m in module.modules():
+            if isinstance(m, nn.Conv2d):
+                return m
+        raise RuntimeError("No Conv2d found in backbone")
+
+    @staticmethod
+    def _replace_first_conv(module: nn.Module, new_conv: nn.Conv2d) -> None:
+        """Replace the first Conv2d in-place (handles resnet and efficientnet)."""
+        for name, child in module.named_children():
+            if isinstance(child, nn.Conv2d):
+                setattr(module, name, new_conv)
+                return
+            try:
+                SegCNN._replace_first_conv(child, new_conv)
+                return
+            except RuntimeError:
+                pass
+        raise RuntimeError("Could not replace first Conv2d")
+
+    # ------------------------------------------------------------------
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        feats = self.backbone(x)
+        if feats.dim() > 2:
+            feats = feats.flatten(1)
+        return self.head(feats)

v3/classes/split_manager.py

@@ -83,6 +83,65 @@ def build_patient_split_plans(
     return plans
 
 
+class EyeLevelSplitManager:
+    """
+    Eye-level (leaky) splitter — splits on individual eye rows, ignoring
+    patient grouping.  Same patient's eyes can appear in different folds.
+    Used to demonstrate the effect of data leakage.
+    """
+
+    def __init__(
+        self,
+        *,
+        patient_col: str = "Patient ID",
+        label_col: Optional[str] = None,
+    ) -> None:
+        self.patient_col = patient_col
+        self.label_col = label_col
+
+    def build_plans(
+        self,
+        *,
+        clinical: Any,
+        args: Any,
+        profile: Optional[Any] = None,
+    ) -> list[PatientSplit]:
+        profile_label_col = getattr(profile, "label_col", None) if profile is not None else None
+        label_col = self.label_col or profile_label_col or getattr(clinical, "label_col", None)
+        if label_col is None:
+            raise ValueError("Could not resolve label column")
+
+        if not hasattr(clinical, "df"):
+            raise ValueError("Clinical object must expose a dataframe at .df")
+        df_full = clinical.df.copy().reset_index(drop=True)
+
+        eval_mode = str(getattr(args, "eval_mode", "multiclass")).lower()
+        if eval_mode == "binary":
+            df_full = df_full[df_full[label_col].isin([0, 1])].reset_index(drop=True)
+
+        n_splits  = int(getattr(args, "n_splits", 5))
+        fold_seed = int(getattr(args, "fold_seed", 42))
+
+        labels = df_full[label_col].to_numpy()
+        eye_ids = df_full.index.to_numpy()
+
+        # Reuse build_patient_split_plans with eye-row IDs as the "patients"
+        plans = build_patient_split_plans(
+            patient_ids=eye_ids,
+            patient_labels=labels,
+            n_splits=n_splits,
+            seed=fold_seed,
+        )
+
+        out: list[PatientSplit] = []
+        for plan in plans:
+            train_df = df_full[df_full.index.isin(plan.train_patient_ids)].reset_index(drop=True)
+            val_df   = df_full[df_full.index.isin(plan.val_patient_ids)].reset_index(drop=True)
+            test_df  = df_full[df_full.index.isin(plan.test_patient_ids)].reset_index(drop=True)
+            out.append(PatientSplit(train=train_df, val=val_df, test=test_df))
+        return out
+
+
 class PatientFirstSplitManager:
     """Patient-level splitter for V3. Outer/inner k-fold, no holdout."""
 

v3/classes/v3_hypertower.py

@@ -23,7 +23,11 @@ from typing import Optional
 import numpy as np
 import torch
 
-from v3.classes.croppers import build_image_preprocessor_from_args
+from v3.classes.croppers import (
+    ManifestImageCropper,
+    UNetImageCropper,
+    build_image_preprocessor_from_args,
+)
 from v3.classes.image_loader import CachedImageLoader
 from v3.classes.dataset import _ClinicalView  # noqa: F401
 from v3.classes.loader_factory import (
@@ -35,10 +39,14 @@ from v3.classes.loader_factory import (
 from v3.classes.metrics import _score_arrays, _svf, _tune_and_snap
 from v3.classes.models import (
     BilateralHT,
+    EmbeddingMLPEnsembleHT,
     FusedEnsembleHT,
+    LogitMLPEnsembleHT,
+    SiameseHT,
     SingleEyeHT,
     V2ModeComparisonOps,
     collect_probs_bilateral,
+    collect_probs_siamese,
     collect_probs_bilateral_components,
     collect_probs_classic,
     collect_probs_ensemble,
@@ -47,6 +55,7 @@ from v3.classes.models import (
     collect_probs_fused,
     collect_probs_single_components,
     train_bilateral_epoch,
+    train_siamese_epoch,
     train_fusion_epoch,
     train_single_epoch,
 )
@@ -54,7 +63,7 @@ from v3.classes.papila_builders import build_papila_data
 from v3.classes.predictions import PredictionStore, head_names_for_mode
 from v3.classes.profiles import build_papila_profile
 from v3.classes.results import FoldArtifacts, FoldResult, _f, _nan, _sv
-from v3.classes.split_manager import PatientFirstSplitManager
+from v3.classes.split_manager import EyeLevelSplitManager, PatientFirstSplitManager
 from v3.classes.transforms import build_eval_transform
 from v3.classes.utils import (
     _drop_mixed_label_patients,
@@ -145,11 +154,14 @@ class V3HyperTower:
         ap.add_argument("--exclude-cols", nargs="*", default=[])
         ap.add_argument("--eval-mode",    choices=["binary", "multiclass"], default="binary")
         ap.add_argument(
-            "--tower-mode", choices=["single", "ensemble", "bilateral", "classic"],
+            "--tower-mode", choices=["single", "ensemble", "bilateral", "siamese", "classic"],
             default="ensemble",
         )
         ap.add_argument("--n-splits",  type=int, default=5)
         ap.add_argument("--fold-seed", type=int, default=42)
+        ap.add_argument("--leaky-cv",  action="store_true",
+                        help="Split at eye level (leaky: same patient can span folds). "
+                             "Used to demonstrate data-leakage effect.")
         ap.add_argument(
             "--folds", type=int, default=None,
             help="Optional cap on number of folds to run.",
@@ -157,9 +169,9 @@ class V3HyperTower:
         ap.add_argument("--epochs",               type=int,   default=40)
         ap.add_argument("--warmup-tower-epochs",  type=int,   default=None)
         ap.add_argument("--warmup-fused-epochs",  type=int,   default=None)
-        ap.add_argument("--single-warmup-tower-epochs", type=int, default=None)
+        ap.add_argument("--single-warmup-tower-epochs", type=int, default=3)
-        ap.add_argument("--single-warmup-fused-epochs", type=int, default=None)
+        ap.add_argument("--single-warmup-fused-epochs", type=int, default=3)
-        ap.add_argument("--warmup-cd-epochs",     type=int,   default=0)
+        ap.add_argument("--warmup-cd-epochs",     type=int,   default=40)
         ap.add_argument("--bilat-warmup-tower-epochs", type=int, default=None)
         ap.add_argument("--bilat-warmup-fused-epochs", type=int, default=None)
         ap.add_argument("--batch-size",           type=int,   default=8)
@@ -170,7 +182,7 @@ class V3HyperTower:
         ap.add_argument("--freeze-ratio",         type=float, default=0.0)
         ap.add_argument("--augment",              action="store_true")
         ap.add_argument("--balanced-sampling",    action="store_true")
-        ap.add_argument("--num-workers",          type=int,   default=4)
+        ap.add_argument("--num-workers",          type=int,   default=8)
         ap.add_argument("--in-memory-cache",      action="store_true", default=True)
         ap.add_argument("--no-in-memory-cache",   action="store_false", dest="in_memory_cache")
         ap.add_argument("--cache-workers",        type=int,   default=4)
@@ -191,10 +203,20 @@ class V3HyperTower:
         ap.add_argument("--img-crop-cache",      type=str,   default="cache_data/hypertower_crops")
         ap.add_argument("--persist-img-crop-cache", action="store_true")
         # Architecture
-        ap.add_argument("--cd-hidden-dim", type=int, default=128)
+        ap.add_argument("--cd-hidden-dim",   type=int,   default=128)
-        ap.add_argument("--fusion-dim",    type=int, default=256)
+        ap.add_argument("--fusion-dim",      type=int,   default=256)
-        ap.add_argument("--bridge-mode",   default="fused",
+        ap.add_argument("--bridge-mode",     default="fused",
                         choices=["fused", "image_only", "clinical_only"])
+        ap.add_argument("--bridge-dropout",  type=float, default=0.5,
+                        help="Dropout in bridge classifier_fused (default: 0.5)")
+        ap.add_argument("--cd-dropout",      type=float, default=0.1,
+                        help="Dropout in clinical tower MLP (default: 0.1)")
+        ap.add_argument("--se-img-tower",    action="store_true",
+                        help="Enable SE gate on image tower output features")
+        ap.add_argument("--se-cd-tower",     action="store_true",
+                        help="Enable SE gate on clinical tower output features")
+        ap.add_argument("--se-bridge",       action="store_true",
+                        help="Enable SE gate on fused vector inside the bridge")
         # Mixed patients
         ap.add_argument("--exclude-mixed-patients",      dest="exclude_mixed_patients",
                         action="store_true")
@@ -217,6 +239,19 @@ class V3HyperTower:
         # Fused head
         ap.add_argument("--fused-head",     action="store_true")
         ap.add_argument("--fusion-epochs",  type=int, default=10)
+        ap.add_argument("--head-type",
+                        choices=["attention", "logit_mlp", "embedding_mlp"],
+                        default="attention",
+                        help="Which bilateral head to train on top of frozen ensemble base")
+        ap.add_argument("--save-checkpoints", action="store_true",
+                        help="Save best_single.pt per fold for explainability / GradCAM")
+        # Geometry features
+        ap.add_argument("--geometry-dim", type=int, default=0,
+                        help="Append N geometry features to clinical metadata (0=disabled, 5=all). "
+                             "Requires --img-crop-manifest.")
+        ap.add_argument("--geometry-source", default="gt", choices=["gt", "unet"],
+                        help="Source for geometry features: gt (GT contour annotations) or "
+                             "unet (U-Net segmentation). unet also requires --img-crop-weights.")
         return ap
 
     def __init__(self, args) -> None:
@@ -246,6 +281,49 @@ class V3HyperTower:
             patient_col="Patient ID", label_col=args.label_col, sample_mode="patient"
         )
 
+        # Build geometry provider if requested, extend feature_dim to include geometry.
+        # Both ManifestImageCropper and UNetImageCropper already have geometry_features()
+        # and precompute_geometry() — we just pick the right one and pre-compute upfront.
+        self.geometry_provider = None
+        geom_dim = int(getattr(args, "geometry_dim", 0))
+        if geom_dim > 0:
+            source = getattr(args, "geometry_source", "gt")
+            manifest = getattr(args, "img_crop_manifest", None)
+            if not manifest:
+                raise ValueError("--geometry-dim requires --img-crop-manifest")
+            all_paths = [
+                self.data.get_image_path(row)
+                for _, row in self.data.df.iterrows()
+            ]
+            if source == "gt":
+                # Reuse image_preprocessor if it's already a ManifestImageCropper,
+                # otherwise build a lightweight one just for geometry (no crop cache).
+                if isinstance(self.image_preprocessor, ManifestImageCropper):
+                    provider = self.image_preprocessor
+                else:
+                    provider = ManifestImageCropper(manifest_path=Path(manifest))
+                print(f"[geometry] GT source — pre-computing geometry from {manifest}", flush=True)
+            elif source == "unet":
+                weights = getattr(args, "img_crop_weights", None)
+                if not weights:
+                    raise ValueError("--geometry-source unet requires --img-crop-weights")
+                if isinstance(self.image_preprocessor, UNetImageCropper):
+                    provider = self.image_preprocessor
+                else:
+                    provider = UNetImageCropper(
+                        manifest_path=Path(manifest),
+                        weights_path=Path(weights),
+                        normalize=getattr(args, "img_crop_normalize", "per_image"),
+                        threshold=getattr(args, "img_crop_threshold", 0.5),
+                    )
+                print(f"[geometry] UNet source — pre-computing geometry from {weights}", flush=True)
+            else:
+                raise ValueError(f"Unknown --geometry-source: {source!r}")
+            provider.precompute_geometry(all_paths)
+            self.geometry_provider = provider
+            self.data.feature_dim += geom_dim
+            print(f"[geometry] feature_dim extended to {self.data.feature_dim} (+{geom_dim} geometry)", flush=True)
+
     def run(self) -> Path:
         """Execute the full fold loop."""
         args = self.args
@@ -276,7 +354,11 @@ class V3HyperTower:
         num_classes = 2 if mode == "binary" else int(df_mode[args.label_col].nunique())
         print(f"\n[{mode}] num_classes={num_classes}  rows={len(df_mode)}  patients={df_mode['Patient ID'].nunique()}", flush=True)
 
-        split_manager = PatientFirstSplitManager(patient_col="Patient ID", label_col=args.label_col)
+        if getattr(args, "leaky_cv", False):
+            split_manager = EyeLevelSplitManager(patient_col="Patient ID", label_col=args.label_col)
+            print("[CV] WARNING: leaky-cv mode — eye-level splits, same patient can span folds.", flush=True)
+        else:
+            split_manager = PatientFirstSplitManager(patient_col="Patient ID", label_col=args.label_col)
         split_args = SimpleNamespace(
             eval_mode=mode,
             n_splits=args.n_splits,
@@ -390,7 +472,7 @@ class V3HyperTower:
             _test_key = "classic_test"
         elif tower_mode == "ensemble":
             _test_key = "ensemble_test"
-        elif tower_mode == "bilateral":
+        elif tower_mode in ("bilateral", "siamese"):
             _test_key = "bilat_test"
         else:
             _test_key = "classic_test"
@@ -400,6 +482,25 @@ class V3HyperTower:
 
         return out_dir
 
+    def _augment_geometry(self, samples: list) -> list:
+        """Append geometry features to matrix_1/matrix_2 in each sample dict."""
+        if self.geometry_provider is None:
+            return samples
+        geom_dim = int(getattr(self.args, "geometry_dim", 0))
+        for s in samples:
+            for img_slot, mat_slot in (("image_1", "matrix_1"), ("image_2", "matrix_2")):
+                img_path = s.get(img_slot)
+                mat = s.get(mat_slot)
+                if img_path is None or mat is None:
+                    continue
+                vec = self.geometry_provider.geometry_for_image(img_path)
+                if vec is not None and len(vec) >= geom_dim:
+                    geom = vec[:geom_dim].astype(np.float32)
+                else:
+                    geom = np.zeros(geom_dim, dtype=np.float32)
+                s[mat_slot] = np.concatenate([np.asarray(mat, dtype=np.float32), geom])
+        return samples
+
     def _run_fold(
         self,
         *,
@@ -420,9 +521,10 @@ class V3HyperTower:
         image_preprocessor = self.image_preprocessor
         nan = float("nan")
 
-        run_single = tower_mode in ("single", "ensemble")
+        run_single  = tower_mode in ("single", "ensemble")
-        run_bilat  = tower_mode == "bilateral"
+        run_bilat   = tower_mode == "bilateral"
-        run_fused  = tower_mode == "ensemble" and getattr(args, "fused_head", False)
+        run_siamese = tower_mode == "siamese"
+        run_fused   = tower_mode == "ensemble" and getattr(args, "fused_head", False)
 
         # ---- warmup schedule -------------------------------------------
         global_warmup_tower = getattr(args, "warmup_tower_epochs", None)
@@ -442,7 +544,7 @@ class V3HyperTower:
         single_warmup_cd = int(getattr(args, "warmup_cd_epochs", 0)) if run_single else 0
         if not run_single:
             single_warmup_tower = single_warmup_fused = 0
-        if not run_bilat:
+        if not run_bilat and not run_siamese:
             bilat_warmup_tower = bilat_warmup_fused = 0
         # Warmup is meaningless in single-pathway modes — skip it entirely
         _bridge_mode = getattr(args, "bridge_mode", "fused")
@@ -451,7 +553,7 @@ class V3HyperTower:
             bilat_warmup_tower = bilat_warmup_fused = 0
         main_epochs = int(args.epochs)
         total_single_epochs = (single_warmup_cd + single_warmup_tower + single_warmup_fused + main_epochs) if run_single else 0
-        total_bilat_epochs  = (bilat_warmup_tower + bilat_warmup_fused + main_epochs) if run_bilat else 0
+        total_bilat_epochs  = (bilat_warmup_tower + bilat_warmup_fused + main_epochs) if (run_bilat or run_siamese) else 0
         total_epochs = max(total_single_epochs, total_bilat_epochs)
 
         # ---- samples ---------------------------------------------------
@@ -460,6 +562,12 @@ class V3HyperTower:
         bilat_val   = filter_bilateral_samples(profile_patient.build_samples(df=split.val,   clinical=data))
         bilat_test  = filter_bilateral_samples(profile_patient.build_samples(df=split.test,  clinical=data)) if split.test is not None else []
 
+        if self.geometry_provider is not None:
+            eye_train   = self._augment_geometry(eye_train)
+            bilat_train = self._augment_geometry(bilat_train)
+            bilat_val   = self._augment_geometry(bilat_val)
+            bilat_test  = self._augment_geometry(bilat_test)
+
         if pred_store is not None:
             if tower_mode in ("single", "classic"):
                 train_sids = [f"{s['id_1']}{s.get('eye_id_1','')}" for s in eye_train]
@@ -502,27 +610,41 @@ class V3HyperTower:
                 y_true_bilat=None, probs_bilat=None,
             )
 
-        # ---- models ----------------------------------------------------
+        # ---- models (CPU for now — moved to device after workers spawn) ---
         single = None
         bilateral = None
+        siamese = None
         if run_single:
             single = SingleEyeHT(
                 backbone=args.backbone, freeze_ratio=args.freeze_ratio,
                 augment=args.augment, clinical_data=data, num_classes=num_classes,
                 cd_hidden_dim=args.cd_hidden_dim, fusion_dim=args.fusion_dim,
                 bridge_mode=getattr(args, "bridge_mode", "fused"),
-            ).to(device)
+                bridge_dropout=getattr(args, "bridge_dropout", 0.5),
+                cd_dropout=getattr(args, "cd_dropout", 0.1),
+                se_img_tower=getattr(args, "se_img_tower", False),
+                se_cd_tower=getattr(args, "se_cd_tower", False),
+                se_bridge=getattr(args, "se_bridge", False),
+            )
         if run_bilat:
             bilateral = BilateralHT(
                 backbone=args.backbone, freeze_ratio=args.freeze_ratio,
                 augment=args.augment, clinical_data=data, num_classes=num_classes,
                 cd_hidden_dim=args.cd_hidden_dim, fusion_dim=args.fusion_dim,
-            ).to(device)
+            )
+        if run_siamese:
+            siamese = SiameseHT(
+                backbone=args.backbone, freeze_ratio=args.freeze_ratio,
+                augment=args.augment, num_classes=num_classes,
+                fusion_dim=args.fusion_dim,
+            )
 
         slots_eye     = profile_eye.slot_descriptors()
         slots_patient = profile_patient.slot_descriptors()
+        _persistent_workers = args.num_workers > 0
         loader_kw     = dict(batch_size=args.batch_size, num_workers=args.num_workers,
-                             image_cache=image_cache)
+                             image_cache=image_cache,
+                             persistent_workers=_persistent_workers)
 
         # ---- loaders ---------------------------------------------------
         use_balanced = bool(getattr(args, "balanced_sampling", False))
@@ -553,6 +675,13 @@ class V3HyperTower:
                 image_preprocessor=image_preprocessor, shuffle=True,
                 sampler=bilat_sampler, **loader_kw,
             )
+        elif run_siamese:
+            siamese_sampler = build_balanced_sampler(bilat_train) if use_balanced else None
+            train_bilat_loader = make_loader(
+                bilat_train, slots_patient, image_transform=siamese.transform,
+                image_preprocessor=image_preprocessor, shuffle=True,
+                sampler=siamese_sampler, **loader_kw,
+            )
         elif run_fused:
             fused_sampler = build_balanced_sampler(bilat_train) if use_balanced else None
             train_bilat_loader = make_loader(
@@ -581,8 +710,26 @@ class V3HyperTower:
             if _ldr is not None:
                 _ldr.dataset.prebuild_image_cache()
 
-        opt_single    = torch.optim.Adam(single.parameters(),    lr=args.lr) if run_single else None
+        # ---- spawn DataLoader workers BEFORE CUDA init -----------------
-        opt_bilateral = torch.optim.Adam(bilateral.parameters(), lr=args.lr) if run_bilat  else None
+        # Workers fork here (clean process state, no CUDA context yet).
+        # persistent_workers=True keeps them alive so the training loop
+        # reuses them rather than re-forking after .to(device).
+        if _persistent_workers:
+            for _ldr in [train_single_loader, train_bilat_loader, val_loader, test_loader]:
+                if _ldr is not None:
+                    _ = iter(_ldr)  # triggers fork now, before CUDA
+
+        # ---- move models to device (CUDA init happens here) ------------
+        if single is not None:
+            single = single.to(device)
+        if bilateral is not None:
+            bilateral = bilateral.to(device)
+        if siamese is not None:
+            siamese = siamese.to(device)
+
+        opt_single    = torch.optim.Adam(single.parameters(),    lr=args.lr) if run_single   else None
+        opt_bilateral = torch.optim.Adam(bilateral.parameters(), lr=args.lr) if run_bilat    else None
+        opt_siamese   = torch.optim.Adam(siamese.parameters(),   lr=args.lr) if run_siamese  else None
 
         # ---- epoch log -------------------------------------------------
         epoch_fields = [
@@ -679,7 +826,7 @@ class V3HyperTower:
             else:
                 phase_single, main_epoch_single, single_active = "done", main_epochs, False
 
-            if not run_bilat:
+            if not run_bilat and not run_siamese:
                 phase_bilat, main_epoch_bilat, bilat_active = "inactive", 0, False
             elif epoch < bilat_warmup_tower:
                 phase_bilat, main_epoch_bilat, bilat_active = "tower_warmup", 0, True
@@ -709,6 +856,12 @@ class V3HyperTower:
                     phase=phase_bilat, bcd_prob=float(args.bcd_prob),
                     tower_loss_mode=args.tower_loss_mode,
                 )
+            elif run_siamese and bilat_active:
+                bl_loss, bl_acc = train_siamese_epoch(
+                    siamese, train_bilat_loader, opt_siamese, device,
+                    bcd_prob=float(args.bcd_prob),
+                    tower_loss_mode=args.tower_loss_mode,
+                )
             else:
                 bl_loss, bl_acc = nan, nan
 
@@ -763,6 +916,10 @@ class V3HyperTower:
                 bi_acc_cd  = float((p_bi_cd.argmax(1) ==y_bi).mean()) if y_bi.size else nan
                 _, bi_auc_img, _ = _score_arrays(y_bi, p_bi_img, num_classes)
                 _, bi_auc_cd, _  = _score_arrays(y_bi, p_bi_cd,  num_classes)
+            elif run_siamese and not _skip_val_eval:
+                y_bi, p_bi = collect_probs_siamese(siamese, val_loader, device)
+                bi_acc, bi_auc, bi_n = _score_arrays(y_bi, p_bi, num_classes)
+                bi_acc_img = bi_acc_cd = bi_auc_img = bi_auc_cd = nan
             else:
                 y_bi = np.array([], dtype=np.int64)
                 p_bi = np.zeros((0, 0), dtype=np.float32)
@@ -905,7 +1062,7 @@ class V3HyperTower:
                 _bar_w = 30
                 _filled = int(_bar_w * (epoch + 1) / single_warmup_cd)
                 _bar = "#" * _filled + "-" * (_bar_w - _filled)
-                msg = f"  [fold {fold+1}] md_warmup [{_bar}] {epoch+1}/{single_warmup_cd}  loss={sl_loss:.4f}"
+                msg = f"  [fold {fold+1}] md_warmup [{_bar}] {epoch+1}/{single_warmup_cd}  loss={sl_loss:.2f}"
                 print(f"\r{msg}", end="", flush=True)
                 fold_logger.info(msg)
                 _prev_phase_single = phase_single
@@ -917,13 +1074,13 @@ class V3HyperTower:
             if args.log_every > 0 and (epoch + 1) % args.log_every == 0:
                 _epoch_secs = time.time() - _epoch_t0
                 if tower_mode == "ensemble":
-                    msg = f"  ep {epoch+1:>3}/{total_epochs} ({_epoch_secs:.1f}s)  auc={en_auc:.4f}  acc={en_acc:.4f}"
+                    _auc_v, _acc_v = en_auc, en_acc
-                elif tower_mode == "bilateral":
+                elif tower_mode in ("bilateral", "siamese"):
-                    msg = f"  ep {epoch+1:>3}/{total_epochs} ({_epoch_secs:.1f}s)  auc={bi_auc:.4f}  acc={bi_acc:.4f}"
+                    _auc_v, _acc_v = bi_auc, bi_acc
                 else:
-                    msg = f"  ep {epoch+1:>3}/{total_epochs} ({_epoch_secs:.1f}s)  auc={cl_auc:.4f}  acc={cl_acc:.4f}"
+                    _auc_v, _acc_v = cl_auc, cl_acc
-                print(msg, flush=True)
+                print(f"  ep {epoch+1:>3}/{total_epochs} ({_epoch_secs:.1f}s)  auc={_auc_v:.2f}  acc={_acc_v:.2f}", flush=True)
-                fold_logger.info(msg)
+                fold_logger.info(f"  ep {epoch+1:>3}/{total_epochs} ({_epoch_secs:.1f}s)  auc={_auc_v:.4f}  acc={_acc_v:.4f}")
 
             _prev_phase_single = phase_single
 
@@ -958,8 +1115,14 @@ class V3HyperTower:
         if run_fused and single is not None:
             for p in single.parameters():
                 p.requires_grad_(False)
-            fused = FusedEnsembleHT(single, num_classes).to(device)
+            head_type = getattr(args, "head_type", "attention")
-            opt_fused = torch.optim.Adam(fused.eye_scorer.parameters(), lr=args.lr)
+            if head_type == "logit_mlp":
+                fused = LogitMLPEnsembleHT(single, num_classes).to(device)
+            elif head_type == "embedding_mlp":
+                fused = EmbeddingMLPEnsembleHT(single, num_classes).to(device)
+            else:
+                fused = FusedEnsembleHT(single, num_classes).to(device)
+            opt_fused = torch.optim.Adam(fused.head.parameters(), lr=args.lr)
             fusion_epochs = int(getattr(args, "fusion_epochs", 10))
             print(
                 f"  [fold {fold+1}] Phase 2: fusion head  bilat_train_n={len(bilat_train)}  epochs={fusion_epochs}",
@@ -977,8 +1140,8 @@ class V3HyperTower:
                     snap_fused, _, _, _ = _tune_and_snap(y_fu, p_fu, fu_acc_val, num_classes, args, args.ece_bins)
                 if (fep + 1) % max(1, args.log_every) == 0:
                     print(
-                        f"  [fold {fold+1}] fusion ep{fep+1:>3}  loss={fu_loss:.4f}  "
+                        f"  [fold {fold+1}] fusion ep{fep+1:>3}  loss={fu_loss:.2f}  "
-                        f"val_auc={fu_auc:.4f}",
+                        f"val_auc={fu_auc:.2f}",
                         flush=True,
                     )
             # No checkpoint saving for fused head either.
@@ -1015,9 +1178,16 @@ class V3HyperTower:
             p_cl_best_img = p_cl_best_md = None
         if run_bilat:
             y_bi_best, p_bi_best = collect_probs_bilateral(bilateral, val_loader, device)
+        elif run_siamese:
+            y_bi_best, p_bi_best = collect_probs_siamese(siamese, val_loader, device)
         else:
             y_bi_best = p_bi_best = None
 
+        # Optional checkpoint saving (final-epoch weights for explainability)
+        if getattr(args, "save_checkpoints", False) and run_single and single is not None:
+            import torch as _torch
+            _torch.save(single.state_dict(), fold_dir / "best_single.pt")
+
         # Compute val snaps from final-epoch model state
         if run_single and tower_mode == "single" and y_cl_best is not None:
             snap_cl, _, _, _ = _tune_and_snap(y_cl_best, p_cl_best, float((p_cl_best.argmax(1) == y_cl_best).mean()), num_classes, args, args.ece_bins)
@@ -1025,7 +1195,7 @@ class V3HyperTower:
         elif run_single and tower_mode == "ensemble" and y_en_best is not None:
             snap_en, _, _, _ = _tune_and_snap(y_en_best, p_en_best, float((p_en_best.argmax(1) == y_en_best).mean()), num_classes, args, args.ece_bins)
             snap_ensemble = snap_en
-        if run_bilat and y_bi_best is not None:
+        if (run_bilat or run_siamese) and y_bi_best is not None:
             snap_bi, _, _, _ = _tune_and_snap(y_bi_best, p_bi_best, float((p_bi_best.argmax(1) == y_bi_best).mean()), num_classes, args, args.ece_bins)
             snap_bilat = snap_bi
 
@@ -1050,6 +1220,8 @@ class V3HyperTower:
                 y_test_out, p_test_out, _, _ = collect_probs_bilateral_components(
                     bilateral, test_loader, device
                 )
+            elif run_siamese:
+                y_test_out, p_test_out = collect_probs_siamese(siamese, test_loader, device)
             if y_test_out is not None and y_test_out.size:
                 test_acc_raw = float((p_test_out.argmax(1) == y_test_out).mean())
                 snap_test, _, _, _ = _tune_and_snap(
@@ -1057,11 +1229,11 @@ class V3HyperTower:
                 )
                 print(
                     f"  [fold {fold+1}] TEST  "
-                    f"auc={snap_test.get('auc', nan):.4f}  "
+                    f"auc={snap_test.get('auc', nan):.2f}  "
-                    f"acc={snap_test.get('acc', nan):.4f}  "
+                    f"acc={snap_test.get('acc', nan):.2f}  "
-                    f"kappa={snap_test.get('kappa', nan):.4f}  "
+                    f"kappa={snap_test.get('kappa', nan):.2f}  "
-                    f"f1={snap_test.get('macro_f1', nan):.4f}  "
+                    f"f1={snap_test.get('macro_f1', nan):.2f}  "
-                    f"ece={snap_test.get('ece', nan):.4f}  "
+                    f"ece={snap_test.get('ece', nan):.2f}  "
                     f"n={snap_test.get('n', 0)}",
                     flush=True,
                 )
@@ -1118,11 +1290,11 @@ class V3HyperTower:
             classic_test_kappa=snap_test.get("kappa", nan)   if tower_mode == "single"    else nan,
             classic_test_f1=snap_test.get("macro_f1", nan)   if tower_mode == "single"    else nan,
             classic_test_ece=snap_test.get("ece", nan)       if tower_mode == "single"    else nan,
-            bilat_test_auc=snap_test.get("auc", nan)         if tower_mode == "bilateral" else nan,
+            bilat_test_auc=snap_test.get("auc", nan)         if tower_mode in ("bilateral", "siamese") else nan,
-            bilat_test_acc=snap_test.get("acc", nan)         if tower_mode == "bilateral" else nan,
+            bilat_test_acc=snap_test.get("acc", nan)         if tower_mode in ("bilateral", "siamese") else nan,
-            bilat_test_kappa=snap_test.get("kappa", nan)     if tower_mode == "bilateral" else nan,
+            bilat_test_kappa=snap_test.get("kappa", nan)     if tower_mode in ("bilateral", "siamese") else nan,
-            bilat_test_f1=snap_test.get("macro_f1", nan)     if tower_mode == "bilateral" else nan,
+            bilat_test_f1=snap_test.get("macro_f1", nan)     if tower_mode in ("bilateral", "siamese") else nan,
-            bilat_test_ece=snap_test.get("ece", nan)         if tower_mode == "bilateral" else nan,
+            bilat_test_ece=snap_test.get("ece", nan)         if tower_mode in ("bilateral", "siamese") else nan,
             test_n=test_n,
             single_train_n=len(eye_train),
             bilat_train_n=len(bilat_train),
@@ -1222,18 +1394,18 @@ class V3HyperTower:
     @staticmethod
     def _print_summary(mode: str, s: dict, tower_mode: str | None = None) -> None:
         def f(v):
-            return "  nan  " if v is None else f"{v:.4f}"
+            return " nan " if v is None else f"{v:.2f}"
         def fsd(mean, std):
-            if mean is None: return "    nan    "
+            if mean is None: return "   nan   "
-            if std is None: return f"{mean:.4f}      "
+            if std is None: return f"{mean:.2f}     "
-            return f"{mean:.4f}±{std:.4f}"
+            return f"{mean:.2f}±{std:.2f}"
 
         # Resolve test key
         if tower_mode in ("single", "classic"):
             test_key = "classic_test"
         elif tower_mode == "ensemble":
             test_key = "ensemble_test"
-        elif tower_mode == "bilateral":
+        elif tower_mode in ("bilateral", "siamese"):
             test_key = "bilat_test"
         else:
             test_key = "classic_test"

v3/distributed/cli.py

@@ -0,0 +1,379 @@
+"""
+HyperTower distributed job CLI — submit jobs, view status.
+
+Usage:
+    # View all connected clients
+    python -m v3.distributed.cli clients
+
+    # View a specific client
+    python -m v3.distributed.cli clients <client_id>
+
+    # View jobs (optionally filter by state)
+    python -m v3.distributed.cli jobs [--state pending|running|done|failed]
+
+    # Submit a job
+    python -m v3.distributed.cli submit \\
+        --run-name phase2/leaky \\
+        -- --bridge-mode image_only --backbone resnet50 --reps 10 ...
+
+    # Submit all jobs from a batch file (JSON)
+    python -m v3.distributed.cli submit-batch jobs.json
+
+    # Cancel a pending job
+    python -m v3.distributed.cli cancel <job_id>
+
+Global flags (can also be set via env vars):
+    --server  HT_SERVER   e.g. http://apollo:8765
+    --token   HT_TOKEN
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import time
+from datetime import datetime, timezone, timedelta
+from typing import Optional
+
+import requests
+
+# ──────────────────────────────────────────────────────────────
+# HTTP helpers
+# ──────────────────────────────────────────────────────────────
+
+class _API:
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self._h = {"x-token": token}
+
+    def get(self, path: str, **params) -> object:
+        r = requests.get(f"{self.base_url}{path}", headers=self._h,
+                         params=params, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+    def post(self, path: str, body: dict) -> object:
+        r = requests.post(f"{self.base_url}{path}", headers=self._h,
+                          json=body, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+    def delete(self, path: str) -> object:
+        r = requests.delete(f"{self.base_url}{path}", headers=self._h, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+
+# ──────────────────────────────────────────────────────────────
+# Formatting helpers
+# ──────────────────────────────────────────────────────────────
+
+def _ago(ts: Optional[str]) -> str:
+    if not ts:
+        return "-"
+    try:
+        dt = datetime.fromisoformat(ts)
+        delta = datetime.now(timezone.utc) - dt
+        secs = int(delta.total_seconds())
+        if secs < 60:
+            return f"{secs}s ago"
+        elif secs < 3600:
+            return f"{secs//60}m ago"
+        else:
+            return f"{secs//3600}h{(secs%3600)//60}m ago"
+    except Exception:
+        return ts
+
+
+def _col(text: str, width: int) -> str:
+    text = str(text) if text is not None else "-"
+    return text[:width].ljust(width)
+
+
+def _table(rows: list[list[str]], headers: list[str]):
+    widths = [max(len(str(r[i])) for r in ([headers] + rows)) for i in range(len(headers))]
+    sep = "  "
+    def _row(r):
+        return sep.join(str(r[i]).ljust(widths[i]) for i in range(len(r)))
+    print(_row(headers))
+    print("-" * (sum(widths) + len(sep) * (len(widths) - 1)))
+    for r in rows:
+        print(_row(r))
+
+
+# ──────────────────────────────────────────────────────────────
+# Subcommands
+# ──────────────────────────────────────────────────────────────
+
+def _clients_table(api: _API):
+    """Render one clients snapshot. Returns the printed lines as a string."""
+    clients = api.get("/clients")
+    if not clients:
+        return "No clients connected."
+    rows = []
+    for c in clients:
+        s = c["status"]
+        parts = []
+        if s.get("rep") is not None:
+            parts.append(f"rep{s['rep']:02d}")
+        if s.get("fold") is not None:
+            parts.append(f"f{s['fold']}")
+        if s.get("epoch") is not None:
+            parts.append(f"ep{s['epoch']}/{s.get('total_epochs','?')}")
+            parts.append(f"auc={s.get('last_auc','?')}")
+        prog = " ".join(parts) if parts else "-"
+        rows.append([
+            c["client_id"],
+            c["hostname"],
+            c["gpu_info"][:30],
+            s["state"],
+            s.get("run_name") or "-",
+            prog,
+            _ago(c["last_seen"]),
+        ])
+    headers = ["ID", "HOST", "GPU", "STATE", "RUN", "PROGRESS", "SEEN"]
+    widths = [max(len(str(r[i])) for r in ([headers] + rows)) for i in range(len(headers))]
+    sep = "  "
+    lines = []
+    lines.append(sep.join(str(h).ljust(widths[i]) for i, h in enumerate(headers)))
+    lines.append("-" * (sum(widths) + len(sep) * (len(widths) - 1)))
+    for r in rows:
+        lines.append(sep.join(str(r[i]).ljust(widths[i]) for i in range(len(r))))
+    return "\n".join(lines)
+
+
+def cmd_clients(api: _API, args):
+    if hasattr(args, "client_id") and args.client_id:
+        data = api.get(f"/clients/{args.client_id}")
+        s = data["status"]
+        print(f"client_id  : {data['client_id']}")
+        print(f"hostname   : {data['hostname']}")
+        print(f"gpu        : {data['gpu_info']}")
+        print(f"last_seen  : {_ago(data['last_seen'])}")
+        print(f"state      : {s['state']}")
+        if s.get("job_id"):
+            print(f"job        : {s['job_id']}  ({s.get('run_name', '')})")
+        if s.get("rep") is not None:
+            print(f"progress   : rep {s['rep']}  fold {s.get('fold', '?')}  "
+                  f"ep {s.get('epoch', '?')}/{s.get('total_epochs', '?')}  "
+                  f"({s.get('last_epoch_secs', '?')}s/ep)  "
+                  f"auc={s.get('last_auc', '?')}")
+        return
+
+    watch = getattr(args, "watch", False)
+    interval = getattr(args, "interval", 5)
+
+    if not watch:
+        print(_clients_table(api))
+        return
+
+    try:
+        while True:
+            now = datetime.now().strftime("%H:%M:%S")
+            print(f"\033[H\033[2J", end="")   # clear screen
+            print(f"HyperTower clients  [{now}]  (Ctrl-C to exit)\n")
+            print(_clients_table(api))
+            time.sleep(interval)
+    except KeyboardInterrupt:
+        print("\nStopped.")
+
+
+def _rep_label(job: dict) -> str:
+    """Extract --rep-index from job args if present."""
+    try:
+        a = json.loads(job["args"]) if isinstance(job.get("args"), str) else job.get("args", [])
+        if "--rep-index" in a:
+            idx = a[a.index("--rep-index") + 1]
+            return f"rep{int(idx):02d}"
+    except Exception:
+        pass
+    return "-"
+
+
+def cmd_jobs(api: _API, args):
+    params = {}
+    if hasattr(args, "state") and args.state:
+        params["state"] = args.state
+    jobs = api.get("/jobs", **params)
+    if not jobs:
+        print("No jobs.")
+        return
+    rows = []
+    for j in jobs:
+        attempts = j.get("attempts", 0)
+        rows.append([
+            j["job_id"][:12],
+            j["run_name"],
+            _rep_label(j),
+            j["state"],
+            f"{attempts}" if attempts else "-",
+            j.get("assigned_to") or "-",
+            _ago(j["created_at"]),
+            _ago(j.get("started_at")),
+            _ago(j.get("completed_at")),
+        ])
+    _table(rows, ["JOB_ID", "RUN_NAME", "REP", "STATE", "TRIES", "CLIENT", "CREATED", "STARTED", "DONE"])
+    pending = sum(1 for j in jobs if j["state"] == "pending")
+    running = sum(1 for j in jobs if j["state"] == "running")
+    done    = sum(1 for j in jobs if j["state"] == "done")
+    failed  = sum(1 for j in jobs if j["state"] == "failed")
+    print(f"\n  {len(jobs)} total  |  {pending} pending  {running} running  {done} done  {failed} failed")
+
+
+def cmd_submit(api: _API, args):
+    body = {
+        "run_name": args.run_name,
+        "module": args.module,
+        "args": args.run_args,
+        "output_dir": args.output_dir,
+        "priority": args.priority,
+    }
+    resp = api.post("/jobs", body)
+    print(f"Queued job {resp['job_id']} ({args.run_name})")
+
+
+def cmd_submit_batch(api: _API, args):
+    with open(args.batch_file) as f:
+        jobs = json.load(f)
+    for job in jobs:
+        resp = api.post("/jobs", job)
+        print(f"Queued {resp['job_id']}  ({job['run_name']})")
+
+
+def cmd_submit_cv(api: _API, args):
+    """Submit one job per rep, each writing to repNN under the same run-name."""
+    seed_start = args.rep_seed_start
+    seed_step  = args.rep_seed_step
+    submitted  = []
+    for i in range(args.reps):
+        seed = seed_start + i * seed_step
+        rep_args = [a for a in args.run_args
+                    if a not in ("--reps", "--rep-seed-start", "--rep-seed-step")]
+        rep_args += [
+            "--reps", "1",
+            "--rep-seed-start", str(seed),
+            "--rep-index", str(i),
+        ]
+        body = {
+            "run_name": args.run_name,
+            "module": args.module,
+            "args": rep_args,
+            "output_dir": args.output_dir,
+            "priority": args.priority,
+        }
+        resp = api.post("/jobs", body)
+        submitted.append(resp["job_id"])
+        print(f"Queued rep{i:02d}  seed={seed}  job_id={resp['job_id']}")
+    print(f"\n{len(submitted)} jobs queued for run '{args.run_name}'")
+
+
+def cmd_cancel(api: _API, args):
+    resp = api.delete(f"/jobs/{args.job_id}")
+    print(f"Cancelled {args.job_id}" if resp.get("ok") else resp)
+
+
+def cmd_clear(api: _API, args):
+    body: dict = {}
+    if args.all:
+        body["all"] = True
+    elif args.run_name:
+        body["run_name"] = args.run_name
+    else:
+        body["states"] = args.states or ["done", "failed", "cancelled"]
+    resp = api.post("/jobs/clear", body)
+    print(f"Cleared {resp['cleared']} jobs.")
+
+
+# ──────────────────────────────────────────────────────────────
+# Parser
+# ──────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                  formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""),
+                    help="Server URL (or set HT_SERVER)")
+    ap.add_argument("--token", default=os.environ.get("HT_TOKEN", ""),
+                    help="Shared secret (or set HT_TOKEN)")
+
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    # clients
+    p_cl = sub.add_parser("clients", help="List clients or inspect one")
+    p_cl.add_argument("client_id", nargs="?")
+    p_cl.add_argument("--watch", "-w", action="store_true",
+                      help="Live monitoring mode — refresh every --interval seconds")
+    p_cl.add_argument("--interval", "-n", type=int, default=5,
+                      help="Refresh interval in seconds for --watch (default: 5)")
+
+    # jobs
+    p_j = sub.add_parser("jobs", help="List jobs")
+    p_j.add_argument("--state", choices=["pending", "running", "done", "failed", "cancelled"])
+
+    # submit
+    p_s = sub.add_parser("submit", help="Submit a single job")
+    p_s.add_argument("--run-name", required=True)
+    p_s.add_argument("--module", default="v3.scripts.main.run_cv")
+    p_s.add_argument("--output-dir", default="v3/results")
+    p_s.add_argument("--priority", type=int, default=0)
+    p_s.add_argument("run_args", nargs=argparse.REMAINDER,
+                     help="Args after '--' are forwarded to the module")
+
+    # submit-cv
+    p_cv = sub.add_parser("submit-cv",
+                           help="Submit one job per rep (distributed 10x5 etc.)")
+    p_cv.add_argument("--run-name", required=True)
+    p_cv.add_argument("--reps", type=int, required=True)
+    p_cv.add_argument("--rep-seed-start", type=int, default=100)
+    p_cv.add_argument("--rep-seed-step", type=int, default=100)
+    p_cv.add_argument("--module", default="v3.scripts.main.run_cv")
+    p_cv.add_argument("--output-dir", default="v3/results")
+    p_cv.add_argument("--priority", type=int, default=0)
+    p_cv.add_argument("run_args", nargs=argparse.REMAINDER,
+                      help="Args after '--' forwarded to run_cv (omit --reps/--rep-seed-*)")
+
+    # submit-batch
+    p_b = sub.add_parser("submit-batch", help="Submit jobs from a JSON file")
+    p_b.add_argument("batch_file")
+
+    # cancel
+    p_c = sub.add_parser("cancel", help="Cancel a pending job")
+    p_c.add_argument("job_id")
+
+    # clear
+    p_cl = sub.add_parser("clear", help="Delete jobs by run-name, state, or everything")
+    p_cl.add_argument("--run-name", default=None, help="Delete all jobs with this run-name")
+    p_cl.add_argument("--states", nargs="+",
+                      default=None,
+                      choices=["done", "failed", "cancelled", "pending", "running"],
+                      help="Delete jobs in these states (default: done+failed+cancelled)")
+    p_cl.add_argument("--all", action="store_true", help="Delete ALL jobs")
+
+    args = ap.parse_args()
+
+    if not args.server:
+        ap.error("--server is required (or set HT_SERVER)")
+    if not args.token:
+        ap.error("--token is required (or set HT_TOKEN)")
+
+    # strip leading "--" from run_args if present
+    if hasattr(args, "run_args") and args.run_args and args.run_args[0] == "--":
+        args.run_args = args.run_args[1:]
+
+    api = _API(args.server, args.token)
+
+    dispatch = {
+        "clients": cmd_clients,
+        "jobs": cmd_jobs,
+        "submit": cmd_submit,
+        "submit-cv": cmd_submit_cv,
+        "submit-batch": cmd_submit_batch,
+        "cancel": cmd_cancel,
+        "clear": cmd_clear,
+    }
+    dispatch[args.cmd](api, args)
+
+
+if __name__ == "__main__":
+    main()

v3/distributed/client.py

@@ -0,0 +1,435 @@
+"""
+HyperTower distributed job client daemon.
+
+Registers with the server, polls for jobs, syncs code, runs training,
+uploads results, and loops. Parses stdout to stream live status.
+
+Usage:
+    python -m v3.distributed.client \\
+        --server http://apollo:8765 \\
+        --token <secret> \\
+        --server-ssh rpotter@apollo \\
+        --server-path /home/rpotter/hypertower \\
+        [--local-path ~/hypertower] \\
+        [--poll-interval 15]
+
+Compatibility test (verify GPU env, 1 epoch dry-run):
+    python -m v3.distributed.client ... --test [extra run_cv args]
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import shutil
+import signal
+import socket
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+from pathlib import Path
+from typing import Optional
+
+import requests
+
+from .protocol import (
+    JobResult,
+    JobSpec,
+    PollResponse,
+    RegisterRequest,
+    RegisterResponse,
+    StatusPush,
+)
+
+# ──────────────────────────────────────────────────────────────
+# Server HTTP wrapper
+# ──────────────────────────────────────────────────────────────
+
+class _Server:
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self._h = {"x-token": token}
+        self.client_id: str = ""
+
+    def _post(self, path: str, **kw) -> dict:
+        r = requests.post(f"{self.base_url}{path}", headers=self._h, timeout=15, **kw)
+        r.raise_for_status()
+        return r.json()
+
+    def register(self, hostname: str, gpu_info: str) -> str:
+        data = self._post("/register",
+                          json={"hostname": hostname, "gpu_info": gpu_info})
+        self.client_id = data["client_id"]
+        self.hostname = hostname
+        self.gpu_info = gpu_info
+        return self.client_id
+
+    def _reregister(self):
+        try:
+            self._post("/register",
+                       json={"hostname": self.hostname, "gpu_info": self.gpu_info},
+                       params={"reuse_id": self.client_id})
+        except Exception:
+            pass
+
+    def poll(self) -> Optional[JobSpec]:
+        data = self._post("/poll", params={"client_id": self.client_id})
+        if data.get("please_reregister"):
+            self._reregister()
+        return JobSpec(**data["job"]) if data.get("job") else None
+
+    def push_status(self, status: StatusPush):
+        try:
+            r = requests.post(
+                f"{self.base_url}/status/{self.client_id}",
+                json=status.model_dump(),
+                headers=self._h,
+                timeout=5,
+            )
+            if r.ok and r.json().get("please_reregister"):
+                self._reregister()
+        except Exception:
+            pass  # don't crash job on status push failure
+
+    def complete(self, job_id: str, success: bool, error_msg: Optional[str] = None):
+        self._post("/complete",
+                   json={"job_id": job_id, "success": success, "error_msg": error_msg})
+
+
+# ──────────────────────────────────────────────────────────────
+# GPU info
+# ──────────────────────────────────────────────────────────────
+
+def _gpu_info() -> str:
+    # NVIDIA
+    try:
+        out = subprocess.check_output(
+            ["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader"],
+            text=True, stderr=subprocess.DEVNULL,
+        ).strip()
+        if out:
+            return " | ".join(out.splitlines())
+    except Exception:
+        pass
+    # AMD
+    try:
+        out = subprocess.check_output(
+            ["rocm-smi", "--showproductname", "--csv"],
+            text=True, stderr=subprocess.DEVNULL,
+        ).strip().splitlines()
+        names = [l for l in out if l and not l.startswith("device")]
+        if names:
+            return "AMD: " + " | ".join(names)
+    except Exception:
+        pass
+    # AMD fallback
+    try:
+        out = subprocess.check_output(
+            ["rocminfo"],
+            text=True, stderr=subprocess.DEVNULL,
+        )
+        names = [l.split(":", 1)[1].strip() for l in out.splitlines()
+                 if "Marketing Name:" in l]
+        if names:
+            return "AMD: " + " | ".join(names)
+    except Exception:
+        pass
+    return "no-gpu"
+
+
+# ──────────────────────────────────────────────────────────────
+# rsync helpers
+# ──────────────────────────────────────────────────────────────
+
+def _rsync(src: str, dst: str, delete: bool = False):
+    cmd = ["rsync", "-az", "--info=progress2"]
+    if delete:
+        cmd.append("--delete")
+    cmd += [src, dst]
+    subprocess.run(cmd, check=True)
+
+
+def _sync_code(server_ssh: str, server_path: str, local_path: str):
+    """Pull v3/ source from server → local (overwrites local changes)."""
+    src = f"{server_ssh}:{server_path}/v3/"
+    dst = f"{local_path}/v3/"
+    Path(dst).mkdir(parents=True, exist_ok=True)
+    _rsync(src, dst, delete=True)
+
+
+def _upload_results(server_ssh: str, server_path: str, local_path: str,
+                    run_name: str, output_dir: str):
+    src = f"{local_path}/{output_dir}/{run_name}/"
+    dst = f"{server_ssh}:{server_path}/{output_dir}/{run_name}/"
+    # Ensure parent directory exists on server before rsyncing
+    remote_parent = f"{server_path}/{output_dir}/{Path(run_name).parent}"
+    subprocess.run(["ssh", server_ssh, f"mkdir -p '{remote_parent}'"], check=True)
+    _rsync(src, dst)
+
+
+def _clean_local(local_path: str, run_name: str, output_dir: str):
+    target = Path(local_path) / output_dir / run_name
+    if target.exists():
+        shutil.rmtree(target)
+        print(f"[client] cleaned {target}", flush=True)
+
+
+# ──────────────────────────────────────────────────────────────
+# Stdout parsers (match v3_hypertower.py print format)
+# ──────────────────────────────────────────────────────────────
+
+# "  ep   3/ 30 (6.5s)  auc=0.75  acc=0.82"
+_EP_RE = re.compile(
+    r"ep\s+(\d+)/\s*(\d+)\s+\(([0-9.]+)s\)\s+auc=([0-9.nan]+)\s+acc=([0-9.nan]+)"
+)
+# "[binary:single] fold 2/5"
+_FOLD_RE = re.compile(r"fold\s+(\d+)/\d+", re.IGNORECASE)
+# "Rep 3  fold_seed=..." or "Rep 3/10  fold_seed=..."
+_REP_RE = re.compile(r"Rep\s+(\d+)", re.IGNORECASE)
+
+
+def _parse_line(line: str) -> dict:
+    """Return any structured fields found in a stdout line."""
+    out = {}
+    m = _FOLD_RE.search(line)
+    if m:
+        out["fold"] = int(m.group(1))
+    m = _REP_RE.search(line)
+    if m:
+        out["rep"] = int(m.group(1))
+    m = _EP_RE.search(line)
+    if m:
+        out["epoch"] = int(m.group(1))
+        out["total_epochs"] = int(m.group(2))
+        out["last_epoch_secs"] = float(m.group(3))
+        try:
+            out["last_auc"] = float(m.group(4))
+        except ValueError:
+            pass
+    return out
+
+
+# ──────────────────────────────────────────────────────────────
+# Core job runner
+# ──────────────────────────────────────────────────────────────
+
+def _run_job(job: JobSpec, server: _Server,
+             server_ssh: str, server_path: str, local_path: str,
+             no_sync: bool = False, num_workers: Optional[int] = None,
+             extra_args: list[str] | None = None) -> bool:
+    extra_args = extra_args or []
+    # 1. Sync code
+    if no_sync:
+        print(f"[client] skipping sync (--no-sync)", flush=True)
+    else:
+        print(f"[client] syncing v3/ from server...", flush=True)
+        server.push_status(StatusPush(state="syncing", job_id=job.job_id, run_name=job.run_name))
+        _sync_code(server_ssh, server_path, local_path)
+
+    # 2. Launch training subprocess
+    job_args = job.args
+    if num_workers is not None and "--num-workers" not in job_args:
+        job_args = job_args + ["--num-workers", str(num_workers)]
+    cmd = [sys.executable, "-m", job.module] + job_args + extra_args
+    print(f"[client] running: {' '.join(cmd)}", flush=True)
+    server.push_status(StatusPush(state="running", job_id=job.job_id, run_name=job.run_name))
+
+    # Write stdout to a temp file so the child's fd table is unmodified —
+    # this lets Python 3.14's forkserver communicate over its own pipes without
+    # interference. We tail the file from a thread for live status.
+    log_dir = Path(local_path) / "v3" / "distributed" / "logs"
+    log_dir.mkdir(parents=True, exist_ok=True)
+    log_file = log_dir / f"job_{job.job_id}.log"
+    ctx: dict = {}
+
+    def _tail(path: Path):
+        with open(path, "r") as f:
+            while True:
+                raw = f.readline()
+                if raw:
+                    print(raw, end="", flush=True)
+                    info = _parse_line(raw)
+                    ctx.update(info)
+                    if "epoch" in info:
+                        server.push_status(StatusPush(
+                            state="running",
+                            job_id=job.job_id,
+                            run_name=job.run_name,
+                            rep=ctx.get("rep"),
+                            fold=ctx.get("fold"),
+                            epoch=ctx.get("epoch"),
+                            total_epochs=ctx.get("total_epochs"),
+                            last_epoch_secs=ctx.get("last_epoch_secs"),
+                            last_auc=ctx.get("last_auc"),
+                        ))
+                elif proc.poll() is not None:
+                    # Drain any remaining output then exit
+                    for raw in f:
+                        print(raw, end="", flush=True)
+                    break
+                else:
+                    time.sleep(0.05)
+
+    with open(log_file, "w") as logf:
+        proc = subprocess.Popen(
+            cmd,
+            stdout=logf,
+            stderr=logf,
+            cwd=local_path,
+            start_new_session=True,  # own process group so we can kill all workers on failure
+        )
+
+    tailer = threading.Thread(target=_tail, args=(log_file,), daemon=True)
+    tailer.start()
+    proc.wait()
+    tailer.join(timeout=5)
+    log_file.unlink(missing_ok=True)
+
+    success = proc.returncode == 0
+
+    if not success:
+        # Kill any surviving worker processes in the group (e.g. DataLoader workers
+        # that outlived the main process after an OOM kill)
+        try:
+            os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+        except ProcessLookupError:
+            pass  # already gone
+
+    if not success:
+        print(f"[client] job FAILED (rc={proc.returncode})", flush=True)
+        return False
+
+    # 3. Upload results (skip when client IS the server — results already in place)
+    if no_sync:
+        print(f"[client] skipping upload (--no-sync, results already local)", flush=True)
+    else:
+        print(f"[client] uploading results...", flush=True)
+        server.push_status(StatusPush(state="uploading", job_id=job.job_id, run_name=job.run_name))
+        _upload_results(server_ssh, server_path, local_path, job.run_name, job.output_dir)
+
+        # 4. Clean local (only remote clients need cleanup)
+        _clean_local(local_path, job.run_name, job.output_dir)
+    print(f"[client] job {job.job_id} complete.", flush=True)
+    return True
+
+
+# ──────────────────────────────────────────────────────────────
+# Compatibility test
+# ──────────────────────────────────────────────────────────────
+
+def _compat_test(server: _Server, server_ssh: str, server_path: str,
+                 local_path: str, extra_args: list[str], no_sync: bool = False):
+    """Run 1 epoch on 1 rep to verify the env works end-to-end."""
+    print("[client] === compatibility test ===", flush=True)
+    job = JobSpec(
+        job_id="compat-test",
+        run_name="_compat_test",
+        module="v3.scripts.main.run_cv",
+        args=[
+            "--bridge-mode", "image_only",
+            "--backbone", "resnet50",
+            "--epochs", "1",
+            "--reps", "1",
+            "--rep-seed-start", "42",
+            "--in-memory-cache",
+            "--num-workers", "0",
+            "--output-root", "v3/results",
+            "--run-name", "_compat_test",
+        ] + extra_args,
+        output_dir="v3/results",
+    )
+    ok = _run_job(job, server, server_ssh, server_path, local_path,
+                  no_sync=no_sync, extra_args=[a for a in extra_args if a != "--"])
+    # always clean up test results
+    _clean_local(local_path, "_compat_test", "v3/results")
+    if ok:
+        print("[client] compatibility test PASSED ✓", flush=True)
+    else:
+        print("[client] compatibility test FAILED ✗", flush=True)
+    return ok
+
+
+# ──────────────────────────────────────────────────────────────
+# Main daemon
+# ──────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    ap.add_argument("--server", required=True,
+                    help="Server URL, e.g. http://apollo:8765")
+    ap.add_argument("--token", default=os.environ.get("HT_TOKEN", ""),
+                    help="Shared secret (or set HT_TOKEN env var)")
+    ap.add_argument("--server-ssh", required=True,
+                    help="SSH target for rsync, e.g. rpotter@apollo")
+    ap.add_argument("--server-path", required=True,
+                    help="Absolute path to hypertower root on server")
+    ap.add_argument("--local-path",
+                    default=str(Path.home() / "hypertower"),
+                    help="Absolute path to hypertower root on this machine")
+    ap.add_argument("--poll-interval", type=int, default=15,
+                    help="Seconds to wait between polls when idle")
+    ap.add_argument("--num-workers", type=int, default=None,
+                    help="Override DataLoader num_workers for all jobs on this client.")
+    ap.add_argument("--extra-args", nargs=argparse.REMAINDER, default=[],
+                    help="Extra args appended to every job on this client (e.g. --cache-workers 0). "
+                         "Use -- to separate from client args: --extra-args -- --cache-workers 0")
+    ap.add_argument("--no-sync", action="store_true",
+                    help="Skip rsync of v3/ before each job (use when client IS the server)")
+    ap.add_argument("--test", action="store_true",
+                    help="Run 1-epoch compatibility test and exit")
+    ap.add_argument("extra_args", nargs="*",
+                    help="Extra args forwarded to run_cv in --test mode")
+    args = ap.parse_args()
+
+    if not args.token:
+        ap.error("--token is required (or set HT_TOKEN)")
+
+    hostname = socket.gethostname()
+    gpu_info = _gpu_info()
+    server = _Server(args.server, args.token)
+
+    client_id = server.register(hostname, gpu_info)
+    print(f"[client] registered as {client_id} ({hostname} | {gpu_info})", flush=True)
+
+    if args.test:
+        sys.exit(0 if _compat_test(
+            server, args.server_ssh, args.server_path,
+            args.local_path, args.extra_args,
+            no_sync=args.no_sync,
+        ) else 1)
+
+    print(f"[client] polling every {args.poll_interval}s...", flush=True)
+    while True:
+        try:
+            job = server.poll()
+            if job is None:
+                server.push_status(StatusPush(state="idle"))
+                time.sleep(args.poll_interval)
+                continue
+
+            success = _run_job(
+                job, server,
+                args.server_ssh, args.server_path, args.local_path,
+                no_sync=args.no_sync,
+                num_workers=args.num_workers,
+                extra_args=[a for a in args.extra_args if a != "--"],
+            )
+            server.complete(job.job_id, success,
+                            error_msg=None if success else "non-zero exit code")
+            server.push_status(StatusPush(state="idle"))
+
+        except KeyboardInterrupt:
+            print("\n[client] shutting down", flush=True)
+            break
+        except Exception as exc:
+            print(f"[client] error: {exc}", flush=True)
+            time.sleep(args.poll_interval)
+
+
+if __name__ == "__main__":
+    main()

v3/distributed/protocol.py

@@ -0,0 +1,62 @@
+"""Shared data models for server/client communication."""
+from __future__ import annotations
+
+from typing import Optional
+from pydantic import BaseModel
+
+
+class RegisterRequest(BaseModel):
+    hostname: str
+    gpu_info: str
+
+
+class RegisterResponse(BaseModel):
+    client_id: str
+
+
+class StatusPush(BaseModel):
+    state: str                          # idle | syncing | running | uploading | error
+    job_id: Optional[str] = None
+    run_name: Optional[str] = None
+    rep: Optional[int] = None
+    fold: Optional[int] = None
+    epoch: Optional[int] = None
+    total_epochs: Optional[int] = None
+    last_epoch_secs: Optional[float] = None
+    last_auc: Optional[float] = None
+    error: Optional[str] = None
+
+
+class ClientInfo(BaseModel):
+    client_id: str
+    hostname: str
+    gpu_info: str
+    status: StatusPush
+    last_seen: str
+
+
+class JobSpec(BaseModel):
+    job_id: str
+    run_name: str
+    module: str                         # e.g. "v3.scripts.main.run_cv"
+    args: list[str]
+    output_dir: str = "v3/results"      # local subdir for rsync-back
+
+
+class PollResponse(BaseModel):
+    job: Optional[JobSpec] = None
+    please_reregister: bool = False
+
+
+class JobResult(BaseModel):
+    job_id: str
+    success: bool
+    error_msg: Optional[str] = None
+
+
+class JobSubmit(BaseModel):
+    run_name: str
+    module: str = "v3.scripts.main.run_cv"
+    args: list[str]
+    output_dir: str = "v3/results"
+    priority: int = 0

v3/distributed/server.py

@@ -0,0 +1,402 @@
+"""
+HyperTower distributed job server.
+
+Manages a SQLite job queue and a registry of connected clients.
+Clients poll for work, push status updates, and report completion.
+
+Usage:
+    python -m v3.distributed.server --port 8765 --token <secret>
+
+Environment:
+    HT_TOKEN  — fallback if --token is not passed
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sqlite3
+import threading
+import time
+import uuid
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from fastapi import Depends, FastAPI, Header, HTTPException
+import uvicorn
+
+from .protocol import (
+    ClientInfo,
+    JobResult,
+    JobSpec,
+    JobSubmit,
+    PollResponse,
+    RegisterRequest,
+    RegisterResponse,
+    StatusPush,
+)
+
+# ──────────────────────────────────────────────────────────────
+# Global state
+# ──────────────────────────────────────────────────────────────
+
+_TOKEN: str = ""
+_DB_PATH: Path = Path("v3/distributed/jobs.db")
+_CLIENT_TTL: int = 120   # seconds before a client is considered gone
+_MAX_ATTEMPTS: int = 3   # max times a job is retried before being left as failed
+
+_clients: dict[str, ClientInfo] = {}
+_clients_lock = threading.Lock()
+
+
+def _reap_stale_clients():
+    """Background thread: remove silent clients and re-queue their running jobs."""
+    while True:
+        time.sleep(30)
+        cutoff = datetime.now(timezone.utc).timestamp() - _CLIENT_TTL
+
+        # Step 1: evict timed-out clients from registry
+        with _clients_lock:
+            stale = [
+                cid for cid, c in _clients.items()
+                if datetime.fromisoformat(c.last_seen).timestamp() < cutoff
+            ]
+            for cid in stale:
+                print(f"[server] reaped stale client {cid} ({_clients[cid].hostname})", flush=True)
+                del _clients[cid]
+            known_ids = set(_clients.keys())
+
+        # Step 2: reset any running job whose assigned client is no longer known
+        with _db() as conn:
+            rows = conn.execute(
+                "SELECT job_id, assigned_to FROM jobs WHERE state='running'"
+            ).fetchall()
+            for row in rows:
+                if row["assigned_to"] not in known_ids:
+                    conn.execute(
+                        "UPDATE jobs SET state='pending', assigned_to=NULL, started_at=NULL "
+                        "WHERE job_id=?",
+                        (row["job_id"],)
+                    )
+                    print(f"[server] re-queued job {row['job_id']} "
+                          f"(client {row['assigned_to']} unknown)", flush=True)
+
+# ──────────────────────────────────────────────────────────────
+# Database helpers
+# ──────────────────────────────────────────────────────────────
+
+@contextmanager
+def _db():
+    conn = sqlite3.connect(str(_DB_PATH))
+    conn.row_factory = sqlite3.Row
+    try:
+        yield conn
+        conn.commit()
+    finally:
+        conn.close()
+
+
+def _init_db():
+    _DB_PATH.parent.mkdir(parents=True, exist_ok=True)
+    with _db() as conn:
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS jobs (
+                job_id       TEXT PRIMARY KEY,
+                run_name     TEXT NOT NULL,
+                module       TEXT NOT NULL,
+                args         TEXT NOT NULL,   -- JSON list
+                output_dir   TEXT NOT NULL DEFAULT 'v3/results',
+                state        TEXT NOT NULL DEFAULT 'pending',
+                priority     INTEGER NOT NULL DEFAULT 0,
+                assigned_to  TEXT,
+                created_at   TEXT NOT NULL,
+                started_at   TEXT,
+                completed_at TEXT,
+                error_msg    TEXT,
+                attempts     INTEGER NOT NULL DEFAULT 0
+            )
+        """)
+        # Add attempts column to existing DBs that predate this field
+        try:
+            conn.execute("ALTER TABLE jobs ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0")
+        except Exception:
+            pass  # column already exists
+        # Note: running jobs are NOT reset on startup — active clients will re-register
+        # via poll/status and the reaper will clean up any that don't reconnect within TTL.
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _ensure_client(client_id: str, hostname: str = "", gpu_info: str = "") -> bool:
+    """Re-register a client that survived a server restart.
+    Returns True if the client was unknown (placeholder created) so the caller
+    can ask the client to re-register with full info."""
+    if client_id not in _clients:
+        _clients[client_id] = ClientInfo(
+            client_id=client_id,
+            hostname=hostname or client_id,
+            gpu_info=gpu_info or "unknown",
+            status=StatusPush(state="idle"),
+            last_seen=_now(),
+        )
+        print(f"[server] re-registered {client_id} (survived restart)", flush=True)
+        return True
+    return False
+
+# ──────────────────────────────────────────────────────────────
+# FastAPI app
+# ──────────────────────────────────────────────────────────────
+
+app = FastAPI(title="HyperTower Job Server")
+
+
+def _check_token(x_token: str = Header(...)):
+    if x_token != _TOKEN:
+        raise HTTPException(status_code=403, detail="Invalid token")
+
+
+# ── Registration ──────────────────────────────────────────────
+
+@app.post("/register", response_model=RegisterResponse,
+          dependencies=[Depends(_check_token)])
+def register(req: RegisterRequest, reuse_id: Optional[str] = None):
+    with _clients_lock:
+        client_id = reuse_id if (reuse_id and reuse_id in _clients) else str(uuid.uuid4())[:8]
+        existing_status = _clients[client_id].status if client_id in _clients else StatusPush(state="idle")
+        _clients[client_id] = ClientInfo(
+            client_id=client_id,
+            hostname=req.hostname,
+            gpu_info=req.gpu_info,
+            status=existing_status,
+            last_seen=_now(),
+        )
+    action = "re-registered" if reuse_id else "registered"
+    print(f"[server] {action} {client_id} ({req.hostname} | {req.gpu_info})", flush=True)
+    return RegisterResponse(client_id=client_id)
+
+
+# ── Job polling ───────────────────────────────────────────────
+
+@app.post("/poll", response_model=PollResponse,
+          dependencies=[Depends(_check_token)])
+def poll(client_id: str):
+    with _clients_lock:
+        needs_reregister = _ensure_client(client_id)
+        _clients[client_id].last_seen = _now()
+
+    with _db() as conn:
+        row = conn.execute(
+            "SELECT * FROM jobs WHERE state='pending' "
+            "ORDER BY priority DESC, created_at ASC LIMIT 1"
+        ).fetchone()
+
+        if row is None:
+            return PollResponse(job=None, please_reregister=needs_reregister)
+
+        job_id = row["job_id"]
+        # Reset any other running jobs for this client — client can only work on one at a time.
+        # This cleans up orphans left over from server restarts.
+        cur = conn.execute(
+            "UPDATE jobs SET state='pending', assigned_to=NULL, started_at=NULL "
+            "WHERE assigned_to=? AND state='running' AND job_id!=?",
+            (client_id, job_id),
+        )
+        if cur.rowcount:
+            print(f"[server] reset {cur.rowcount} orphaned running job(s) for {client_id}", flush=True)
+        conn.execute(
+            "UPDATE jobs SET state='running', assigned_to=?, started_at=? WHERE job_id=?",
+            (client_id, _now(), job_id),
+        )
+
+    job = JobSpec(
+        job_id=job_id,
+        run_name=row["run_name"],
+        module=row["module"],
+        args=json.loads(row["args"]),
+        output_dir=row["output_dir"],
+    )
+
+    with _clients_lock:
+        _clients[client_id].status = StatusPush(
+            state="syncing", job_id=job_id, run_name=row["run_name"]
+        )
+
+    print(f"[server] dispatched {job_id} ({row['run_name']}) → {client_id}", flush=True)
+    return PollResponse(job=job, please_reregister=needs_reregister)
+
+
+# ── Status ────────────────────────────────────────────────────
+
+@app.post("/status/{client_id}", dependencies=[Depends(_check_token)])
+def push_status(client_id: str, status: StatusPush):
+    with _clients_lock:
+        needs_reregister = _ensure_client(client_id)
+        _clients[client_id].status = status
+        _clients[client_id].last_seen = _now()
+    return {"ok": True, "please_reregister": needs_reregister}
+
+
+@app.get("/clients", dependencies=[Depends(_check_token)])
+def list_clients():
+    with _clients_lock:
+        return list(_clients.values())
+
+
+@app.get("/clients/{client_id}", dependencies=[Depends(_check_token)])
+def get_client(client_id: str):
+    with _clients_lock:
+        if client_id not in _clients:
+            raise HTTPException(status_code=404, detail="Unknown client")
+        return _clients[client_id]
+
+
+# ── Job completion ────────────────────────────────────────────
+
+@app.post("/complete", dependencies=[Depends(_check_token)])
+def complete(result: JobResult):
+    with _db() as conn:
+        if result.success:
+            conn.execute(
+                "UPDATE jobs SET state='done', completed_at=?, error_msg=NULL WHERE job_id=?",
+                (_now(), result.job_id),
+            )
+            print(f"[server] job {result.job_id} → done", flush=True)
+
+            # Auto-clear run if all jobs for this run_name are now done
+            run_row = conn.execute(
+                "SELECT run_name FROM jobs WHERE job_id=?", (result.job_id,)
+            ).fetchone()
+            if run_row:
+                run_name = run_row["run_name"]
+                remaining = conn.execute(
+                    "SELECT COUNT(*) FROM jobs WHERE run_name=? AND state != 'done'",
+                    (run_name,)
+                ).fetchone()[0]
+                if remaining == 0:
+                    total = conn.execute(
+                        "SELECT COUNT(*) FROM jobs WHERE run_name=?", (run_name,)
+                    ).fetchone()[0]
+                    conn.execute("DELETE FROM jobs WHERE run_name=?", (run_name,))
+                    print(f"[server] run '{run_name}' complete ({total} jobs) — cleared", flush=True)
+        else:
+            row = conn.execute(
+                "SELECT attempts FROM jobs WHERE job_id=?", (result.job_id,)
+            ).fetchone()
+            attempts = (row["attempts"] if row else 0) + 1
+            if attempts < _MAX_ATTEMPTS:
+                conn.execute(
+                    "UPDATE jobs SET state='pending', assigned_to=NULL, started_at=NULL, "
+                    "attempts=?, error_msg=? WHERE job_id=?",
+                    (attempts, result.error_msg, result.job_id),
+                )
+                print(f"[server] job {result.job_id} failed (attempt {attempts}/{_MAX_ATTEMPTS}), "
+                      f"re-queuing", flush=True)
+            else:
+                conn.execute(
+                    "UPDATE jobs SET state='failed', completed_at=?, attempts=?, error_msg=? "
+                    "WHERE job_id=?",
+                    (_now(), attempts, result.error_msg, result.job_id),
+                )
+                print(f"[server] job {result.job_id} failed permanently after "
+                      f"{attempts} attempts", flush=True)
+    return {"ok": True}
+
+
+# ── Job queue management ──────────────────────────────────────
+
+@app.post("/jobs", dependencies=[Depends(_check_token)])
+def submit_job(job: JobSubmit):
+    job_id = str(uuid.uuid4())[:12]
+    with _db() as conn:
+        conn.execute(
+            "INSERT INTO jobs "
+            "(job_id, run_name, module, args, output_dir, priority, created_at) "
+            "VALUES (?,?,?,?,?,?,?)",
+            (job_id, job.run_name, job.module, json.dumps(job.args),
+             job.output_dir, job.priority, _now()),
+        )
+    print(f"[server] queued {job_id} ({job.run_name})", flush=True)
+    return {"job_id": job_id}
+
+
+@app.get("/jobs", dependencies=[Depends(_check_token)])
+def list_jobs(state: Optional[str] = None):
+    with _db() as conn:
+        if state:
+            rows = conn.execute(
+                "SELECT * FROM jobs WHERE state=? ORDER BY created_at DESC", (state,)
+            ).fetchall()
+        else:
+            rows = conn.execute(
+                "SELECT * FROM jobs ORDER BY created_at DESC"
+            ).fetchall()
+    return [dict(r) for r in rows]
+
+
+@app.post("/jobs/clear", dependencies=[Depends(_check_token)])
+def clear_jobs(body: dict):
+    with _db() as conn:
+        if body.get("all"):
+            cur = conn.execute("DELETE FROM jobs")
+        elif body.get("run_name"):
+            cur = conn.execute("DELETE FROM jobs WHERE run_name=?", (body["run_name"],))
+        else:
+            states = body.get("states", ["done", "failed", "cancelled"])
+            placeholders = ",".join("?" * len(states))
+            cur = conn.execute(f"DELETE FROM jobs WHERE state IN ({placeholders})", states)
+    print(f"[server] cleared {cur.rowcount} jobs", flush=True)
+    return {"cleared": cur.rowcount}
+
+
+@app.delete("/jobs/{job_id}", dependencies=[Depends(_check_token)])
+def cancel_job(job_id: str):
+    with _db() as conn:
+        conn.execute(
+            "UPDATE jobs SET state='cancelled' WHERE job_id=? AND state='pending'",
+            (job_id,),
+        )
+    return {"ok": True}
+
+
+# ──────────────────────────────────────────────────────────────
+# Entry point
+# ──────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                  formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--port", type=int, default=8765)
+    ap.add_argument("--host", default="0.0.0.0")
+    ap.add_argument("--token", default=os.environ.get("HT_TOKEN", ""),
+                    help="Shared secret (or set HT_TOKEN env var)")
+    ap.add_argument("--db", default="v3/distributed/jobs.db",
+                    help="Path to SQLite job database")
+    ap.add_argument("--client-ttl", type=int, default=120,
+                    help="Seconds of silence before a client is reaped (default: 120)")
+    ap.add_argument("--max-attempts", type=int, default=3,
+                    help="Max times a failed job is retried before being left as failed (default: 3)")
+    args = ap.parse_args()
+
+    if not args.token:
+        ap.error("--token is required (or set HT_TOKEN)")
+
+    global _TOKEN, _DB_PATH, _CLIENT_TTL, _MAX_ATTEMPTS
+    _TOKEN = args.token
+    _DB_PATH = Path(args.db)
+    _CLIENT_TTL = args.client_ttl
+    _MAX_ATTEMPTS = args.max_attempts
+    _init_db()
+
+    reaper = threading.Thread(target=_reap_stale_clients, daemon=True)
+    reaper.start()
+
+    print(f"[server] listening on {args.host}:{args.port}  client_ttl={_CLIENT_TTL}s", flush=True)
+    uvicorn.run(app, host=args.host, port=args.port, log_level="warning")
+
+
+if __name__ == "__main__":
+    main()

v3/distributed/server_cheat_sheet.md

@@ -0,0 +1,122 @@
+# Distributed Server Cheat Sheet
+
+All commands assume server is running on hades at port 8765.
+
+## Start Server
+```bash
+python -m v3.distributed.server --token hypertower
+```
+Run inside tmux so it survives disconnects:
+```bash
+tmux new -s htserver
+python -m v3.distributed.server --token hypertower
+# Ctrl-B D to detach
+tmux attach -t htserver   # reattach later
+```
+
+## Start Clients
+
+**Hades (server-local, no sync):**
+```bash
+python -m v3.distributed.client \
+    --server http://hades:8765 --token hypertower \
+    --server-ssh ignored --server-path ignored \
+    --local-path /home/rpotter/hypertower \
+    --no-sync
+```
+
+**Apollo (remote client):**
+```bash
+python -m v3.distributed.client \
+    --server http://hades:8765 --token hypertower \
+    --server-ssh rpotter@hades \
+    --server-path /home/rpotter/hypertower \
+    --local-path /home/odin/hypertower
+```
+
+---
+
+## Monitoring
+
+**Live client monitor (refreshes every 5s):**
+```bash
+python -m v3.distributed.cli --server http://hades:8765 --token hypertower clients --watch
+```
+
+**Faster refresh:**
+```bash
+python -m v3.distributed.cli --server http://hades:8765 --token hypertower clients --watch --interval 2
+```
+
+**Inspect a single client:**
+```bash
+python -m v3.distributed.cli --server http://hades:8765 --token hypertower clients <client_id>
+```
+
+**View job queue:**
+```bash
+python -m v3.distributed.cli --server http://hades:8765 --token hypertower jobs
+```
+
+**Filter by state:**
+```bash
+python -m v3.distributed.cli --server http://hades:8765 --token hypertower jobs --state pending
+python -m v3.distributed.cli --server http://hades:8765 --token hypertower jobs --state running
+python -m v3.distributed.cli --server http://hades:8765 --token hypertower jobs --state failed
+```
+
+---
+
+## Submitting Jobs
+
+**Phase 3 main grid:**
+```bash
+python -m v3.scripts.main.phase3.dispatch_phase3 \
+    --server http://hades:8765 --token hypertower
+```
+
+**Dry run (check what would be submitted):**
+```bash
+python -m v3.scripts.main.phase3.dispatch_phase3 \
+    --server http://hades:8765 --token hypertower --dry-run
+```
+
+**Other grids:**
+```bash
+python -m v3.scripts.main.phase3.dispatch_phase3 \
+    --server http://hades:8765 --token hypertower \
+    --grid v3/scripts/main/phase3/epoch_grid.json
+
+python -m v3.scripts.main.phase3.dispatch_phase3 \
+    --server http://hades:8765 --token hypertower \
+    --grid v3/scripts/main/phase3/phase35_grid.json
+```
+
+---
+
+## Queue Management
+
+**Clear failed jobs:**
+```bash
+python -m v3.distributed.cli --server http://hades:8765 --token hypertower clear --states failed
+```
+
+**Clear running jobs (orphan cleanup):**
+```bash
+python -m v3.distributed.cli --server http://hades:8765 --token hypertower clear --states running
+```
+
+**Clear all jobs:**
+```bash
+python -m v3.distributed.cli --server http://hades:8765 --token hypertower clear --all
+```
+
+**Clear by run name:**
+```bash
+python -m v3.distributed.cli --server http://hades:8765 --token hypertower clear --run-name phase3/baseline
+```
+
+**Cancel a specific job:**
+```bash
+python -m v3.distributed.cli --server http://hades:8765 --token hypertower cancel <job_id>
+```

v3/docs/phase_plan.md

@@ -0,0 +1,81 @@
+# HyperTower Phase Plan
+
+## Phase 1 — PAPILA Baseline Reproduction
+**Goal:** Reproduce the CNN results reported in the PAPILA paper and establish proper evaluation methodology.
+
+**Covers:**
+- Reproduce PAPILA paper AUC results for VGG16, MobileNetV2, ResNet50, DenseNet121, InceptionV3
+- Present results with 95% CI and boxplot across repeated CV folds
+- **Show effect of data leakage:** compare leaky CV (same patient in train and test) vs proper patient-stratified CV — motivates repeated CV methodology
+
+**Status:** Complete
+
+---
+
+## Phase 2 — Image Preprocessing & Backbone Selection
+**Goal:** Show the effect of image-level design choices on classification performance.
+
+**Covers:**
+- **Show the effect of pre-training:** Refugelike (REFUGE-pretrained) backbone vs standard ImageNet backbones
+- **Show the effect of cropping:** U-Net optic disc crop vs no cropping, and crop scale sensitivity
+
+**Status:** Complete — best config is refugelike backbone, no crop (proper CV)
+
+---
+
+## Phase 3 — Clinical Data Fusion (Single-Eye)
+**Goal:** Show the effect of combining fundus image features with clinical metadata in the single-eye pipeline.
+
+**Covers:**
+- **Show the effect of combining image data with clinical data:** fused bridge vs image-only and clinical-only ablations
+- Architecture search: loss function (BCD vs all-losses), SE attention, IOP correction strategy, feature ablations, network dimensions, dropout, learning rate, warmup strategy, augmentation, balanced sampling
+- Epoch length sensitivity
+
+**Status:** Complete — key findings: IOP ratio correction + drop raw (+1.7%), excluding axial length helps, age is most informative clinical feature, SE adds no benefit, LR very sensitive
+
+---
+
+## Phase 3.5 — Confirmation & Tuning
+**Goal:** Confirm that the top phase 3 findings combine additively, and tune BCD probability with the best IOP preprocessing.
+
+**Covers:**
+- Combine `iop_ratio_drop_raw` (best preprocessing) with `bcd_p07` (best loss setting)
+- Extend BCD probability sweep to p=0.8 and p=0.9 to find the optimum
+
+**Planned approach:**
+- Baseline is phase 3 `iop_ratio_drop_raw` (0.8685 ± 0.011)
+- All runs use best single-eye settings: refugelike, no crop, ratio IOP + drop raw, no axial length
+
+**Status:** Not started
+
+---
+
+## Phase 4 — Dual CNN Architecture (Image Only)
+**Goal:** Show the effect of processing both eyes jointly, and compare bilateral architectures against the single-eye baseline.
+
+**Covers:**
+- **Show the effect of a dual CNN:** bilateral tower (both eyes) vs single-eye tower, image data only
+- **Architecture comparison:** canonical HyperTower bilateral (siamese shared-weight backbone returning mean+delta features) vs independent per-eye processing with late fusion
+
+**Planned approach:**
+- Image-only, no clinical data — isolates the bilateral vision question cleanly
+- Use best image settings from phase 2 (refugelike, no crop)
+- Compare siamese tower, independent bilateral, and single-eye (phase 3 baseline) directly
+
+**Status:** Not started
+
+---
+
+## Phase 5 — Full HyperTower: Bilateral + Clinical Data + Fusion Heads
+**Goal:** Bring together the best bilateral architecture (phase 4) with clinical data fusion (phase 3), and compare prediction aggregation strategies.
+
+**Covers:**
+- **Show the effect of a dual CNN + clinical data:** bilateral tower with fused clinical bridge — the full HyperTower model
+- **Ensemble vs fused head:** patient-level prediction via ensemble (average OD+OS eye-level scores) vs learned fused head trained on top of single-eye scores
+
+**Planned approach:**
+- Use best settings from all prior phases (refugelike, no crop, ratio IOP + drop raw, no axial length)
+- Compare tower modes: single, bilateral, ensemble, fused-head
+- Establish final best configuration as the HyperTower result
+
+**Status:** Not started

v3/figures/explainability/md_importance_phase5.csv

@@ -0,0 +1,10 @@
+feature,mean_importance,std_importance
+Age,0.02684933048095948,0.018982159670576627
+IOP_corr,0.01402488800132739,0.016679352675009966
+Phakic/Pseudophakic,0.006225936127828556,0.007852635891143775
+Gender,0.0026848811162960276,0.010922597349168684
+eyeID,0.0009305775513090246,0.005399435288001892
+Pachymetry,0.0006101177888381434,0.002786215613365408
+dioptre_2,-0.0005330790563094697,0.001828442063194257
+astigmatism,-0.003371613580988547,0.0019910628774032386
+dioptre_1,-0.004100123269320834,0.0075533314694558405

v3/scripts/development/stage_seg_cnn/run_seg_cnn.py

@@ -0,0 +1,595 @@
+#!/usr/bin/env python
+"""
+Segmentation-map CNN — glaucoma grading from disc/cup label maps.
+
+Trains a CNN whose input is the combined optic disc / cup segmentation map
+(pixel values 0=bg, 1=disc_rim, 2=cup) rather than the original fundus image.
+The model must learn structural relationships like cup-to-disc ratio, rim area,
+and cup eccentricity directly from the segmentation geometry.
+
+Segmentation source (--seg-mode):
+  gt    [default]  Rasterise expert contour annotations from the PAPILA manifest.
+  unet             Run a trained UNetSegmenter on the raw fundus image.
+                   Requires --unet-weights.
+
+Cross-validation:
+  5-fold stratified group CV, with both eyes of the same patient always in the
+  same fold (preventing OD/OS leakage).  Folds are built from the PAPILA
+  clinical CSV, then matched to manifest entries by patient ID + eye.
+
+Usage examples:
+  # GT masks, default settings
+  python -m v3.scripts.main.phase_seg_cnn.run_seg_cnn \\
+      --image-dir   Papila/FundusImages \\
+      --clinical-dir Papila/ClinicalData \\
+      --manifest     manifest.csv
+
+  # U-Net predicted masks, resnet50 backbone
+  python -m v3.scripts.main.phase_seg_cnn.run_seg_cnn \\
+      --image-dir   Papila/FundusImages \\
+      --clinical-dir Papila/ClinicalData \\
+      --manifest     manifest.csv \\
+      --seg-mode unet --unet-weights models/unet_segmenter/best.pt \\
+      --backbone resnet50
+
+  # Single-channel label map instead of one-hot
+  python -m v3.scripts.main.phase_seg_cnn.run_seg_cnn \\
+      --image-dir   Papila/FundusImages \\
+      --clinical-dir Papila/ClinicalData \\
+      --manifest     manifest.csv \\
+      --channels 1 --no-pretrained
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+import time
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+from sklearn.metrics import accuracy_score, roc_auc_score, roc_curve
+from sklearn.model_selection import StratifiedGroupKFold
+from torch.utils.data import DataLoader
+from tqdm import tqdm
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
+
+from v3.classes.seg_cnn import (
+    SegCNN, SegMapDataset, SegMapRecord,
+    UNetFineTuneDataset, precompute_unet_seg_maps,
+)
+
+
+# ---------------------------------------------------------------------------
+# Clinical data loader (PAPILA)
+# ---------------------------------------------------------------------------
+
+def load_papila_labels(
+    clinical_dir: Path,
+    label_col: str = "Diagnosis",
+    drop_suspects: bool = True,
+) -> pd.DataFrame:
+    """
+    Return a DataFrame with columns:
+      patient_id (int), eye (str), label (int)
+
+    Reads patient_data_od.xlsx + patient_data_os.xlsx from clinical_dir.
+    """
+    od_path = clinical_dir / "patient_data_od.xlsx"
+    os_path = clinical_dir / "patient_data_os.xlsx"
+    frames = []
+    for path, eye in ((od_path, "OD"), (os_path, "OS")):
+        if not path.exists():
+            raise FileNotFoundError(f"Clinical data not found: {path}")
+        df = pd.read_excel(path, header=1)
+        df["eye"] = eye
+        id_col = "Patient ID" if "Patient ID" in df.columns else "ID"
+        df["patient_id"] = (
+            df[id_col].astype(str).str.extract(r"(\d+)")[0].astype(int)
+        )
+        frames.append(df)
+    df = pd.concat(frames, ignore_index=True)
+
+    if drop_suspects:
+        df = df[df[label_col] != 2].reset_index(drop=True)
+
+    df["label"] = df[label_col].astype(int)
+    return df[["patient_id", "eye", "label"]].copy()
+
+
+# ---------------------------------------------------------------------------
+# Build SegMapRecords from manifest + clinical labels
+# ---------------------------------------------------------------------------
+
+def build_records(
+    manifest_path: Path,
+    clinical_df: pd.DataFrame,
+    dataset_filter: str = "papila",
+) -> List[SegMapRecord]:
+    """
+    Join manifest entries (image paths + annotation paths) with clinical labels.
+
+    Returns one SegMapRecord per matched eye sample.
+    """
+    manifest = pd.read_csv(manifest_path)
+    papila_rows = manifest[manifest["dataset"] == dataset_filter].copy()
+
+    # Parse patient_id and eye from sample_id  e.g. "papila_RET042OD" → 42, "OD"
+    def _parse(sid: str):
+        sid = sid.replace(f"{dataset_filter}_RET", "")
+        eye = sid[-2:].upper()        # "OD" or "OS"
+        pid = int(sid[:-2])
+        return pid, eye
+
+    papila_rows[["patient_id", "eye"]] = pd.DataFrame(
+        papila_rows["sample_id"].apply(_parse).tolist(),
+        index=papila_rows.index,
+    )
+
+    merged = papila_rows.merge(
+        clinical_df[["patient_id", "eye", "label"]],
+        on=["patient_id", "eye"],
+        how="inner",
+    )
+
+    records: List[SegMapRecord] = []
+    for _, row in merged.iterrows():
+        records.append(
+            SegMapRecord(
+                sample_id=row["sample_id"],
+                image_path=Path(row["image_path"]),
+                annotation_disc=Path(row["annotation_disc"]),
+                annotation_cup=Path(row["annotation_cup"]),
+                annotation_type_disc=row["annotation_type_disc"],
+                annotation_type_cup=row["annotation_type_cup"],
+                patient_id=int(row["patient_id"]),
+                eye=str(row["eye"]),
+                label=int(row["label"]),
+            )
+        )
+    return records
+
+
+# ---------------------------------------------------------------------------
+# Training / evaluation helpers
+# ---------------------------------------------------------------------------
+
+def train_epoch(
+    model: nn.Module,
+    loader: DataLoader,
+    optimizer: torch.optim.Optimizer,
+    criterion: nn.Module,
+    device: torch.device,
+) -> float:
+    model.train()
+    total_loss = 0.0
+    n = 0
+    for x, y in loader:
+        x, y = x.to(device), y.to(device)
+        optimizer.zero_grad()
+        logits = model(x)
+        loss = criterion(logits, y)
+        loss.backward()
+        optimizer.step()
+        total_loss += loss.item() * x.size(0)
+        n += x.size(0)
+    return total_loss / n if n else float("nan")
+
+
+@torch.no_grad()
+def evaluate(
+    model: nn.Module,
+    loader: DataLoader,
+    device: torch.device,
+) -> Tuple[float, float, np.ndarray, np.ndarray]:
+    """Returns (auc, acc, y_true, y_prob)."""
+    model.eval()
+    probs_list, labels_list = [], []
+    for x, y in loader:
+        x = x.to(device)
+        logits = model(x)
+        prob = torch.softmax(logits, dim=1)[:, 1].cpu().numpy()
+        probs_list.append(prob)
+        labels_list.append(y.numpy())
+    y_true = np.concatenate(labels_list)
+    y_prob = np.concatenate(probs_list)
+    auc = float(roc_auc_score(y_true, y_prob)) if len(np.unique(y_true)) > 1 else float("nan")
+    acc = float(accuracy_score(y_true, (y_prob >= 0.5).astype(int)))
+    return auc, acc, y_true, y_prob
+
+
+# ---------------------------------------------------------------------------
+# U-Net fine-tuning
+# ---------------------------------------------------------------------------
+
+def finetune_unet(
+    segmenter,
+    records: List[SegMapRecord],
+    epochs: int,
+    lr: float,
+    batch_size: int,
+    device: torch.device,
+) -> None:
+    """Fine-tune the U-Net on a fold's training records using GT annotations."""
+    import copy
+    ds = UNetFineTuneDataset(
+        records,
+        target_size=segmenter.target_size,
+        normalize=segmenter.normalize,
+    )
+    loader = DataLoader(ds, batch_size=batch_size, shuffle=True, num_workers=0)
+    optimizer = torch.optim.Adam(segmenter.model.parameters(), lr=lr)
+    criterion = nn.BCEWithLogitsLoss()
+
+    segmenter.model.train()
+    for epoch in tqdm(range(1, epochs + 1), desc="  U-Net finetune", unit="ep", leave=False):
+        for images, masks in loader:
+            images, masks = images.to(device), masks.to(device)
+            optimizer.zero_grad()
+            loss = criterion(segmenter.model(images), masks)
+            loss.backward()
+            optimizer.step()
+    segmenter.model.eval()
+
+
+# ---------------------------------------------------------------------------
+# Cross-validation loop
+# ---------------------------------------------------------------------------
+
+def _train_one_split(
+    train_recs, val_recs, y_train, args, device, out_dir, label,
+    train_seg_maps=None, val_seg_maps=None,
+):
+    """Train one fold/split, return (auc, acc, y_true, y_prob, fpr, tpr)."""
+    train_ds = SegMapDataset(
+        train_recs,
+        target_size=args.img_size,
+        channels=args.channels,
+        augment=True,
+        seg_target_size=args.seg_size,
+        crop_to_disc=not args.no_crop,
+        precomputed_seg_maps=train_seg_maps,
+    )
+    val_ds = SegMapDataset(
+        val_recs,
+        target_size=args.img_size,
+        channels=args.channels,
+        augment=False,
+        seg_target_size=args.seg_size,
+        crop_to_disc=not args.no_crop,
+        precomputed_seg_maps=val_seg_maps,
+    )
+    train_loader = DataLoader(
+        train_ds, batch_size=args.batch_size, shuffle=True,
+        num_workers=args.workers, pin_memory=True,
+    )
+    val_loader = DataLoader(
+        val_ds, batch_size=args.batch_size, shuffle=False,
+        num_workers=args.workers, pin_memory=True,
+    )
+
+    model = SegCNN(
+        num_classes=2,
+        backbone=args.backbone,
+        pretrained=not args.no_pretrained,
+        in_channels=args.channels,
+        dropout=args.dropout,
+    ).to(device)
+
+    n_pos = int(y_train.sum())
+    n_neg = len(y_train) - n_pos
+    class_weights = (
+        torch.tensor([1.0, n_neg / n_pos], dtype=torch.float32).to(device)
+        if n_pos > 0 and n_neg > 0 else None
+    )
+    criterion = nn.CrossEntropyLoss(weight=class_weights)
+    optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.wd)
+    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
+        optimizer, T_max=args.epochs, eta_min=args.lr * 0.01
+    )
+
+    best_auc, best_state = -1.0, None
+    epoch_bar = tqdm(range(1, args.epochs + 1), desc=label, unit="ep")
+    for _ in epoch_bar:
+        train_loss = train_epoch(model, train_loader, optimizer, criterion, device)
+        val_auc, val_acc, _, _ = evaluate(model, val_loader, device)
+        scheduler.step()
+        epoch_bar.set_postfix(loss=f"{train_loss:.4f}", auc=f"{val_auc:.3f}", acc=f"{val_acc:.3f}")
+        if val_auc > best_auc:
+            best_auc = val_auc
+            best_state = {k: v.clone() for k, v in model.state_dict().items()}
+            torch.save({"model": best_state, "auc": best_auc}, out_dir / "best.pt")
+
+    if best_state is not None:
+        model.load_state_dict(best_state)
+    final_auc, final_acc, y_true, y_prob = evaluate(model, val_loader, device)
+    fpr, tpr, _ = roc_curve(y_true, y_prob, pos_label=1)
+    return final_auc, final_acc, y_true, y_prob, fpr, tpr
+
+
+def run_cv(
+    records: List[SegMapRecord],
+    args,
+    device: torch.device,
+    unet_segmenter=None,
+    out_dir: Path = Path("analysis_data/seg_cnn"),
+) -> pd.DataFrame:
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    y = np.array([r.label for r in records])
+    groups = np.array([r.patient_id for r in records])
+    splits = list(StratifiedGroupKFold(n_splits=args.n_splits).split(
+        np.arange(len(records)), y, groups
+    ))
+
+    # If using U-Net without fine-tuning, precompute all seg maps once upfront.
+    # If fine-tuning, we must precompute per fold (after fine-tuning) so we
+    # save the base weights here to restore at the start of each fold.
+    base_unet_state = None
+    all_seg_maps = None
+    if unet_segmenter is not None:
+        if args.finetune_epochs > 0:
+            import copy
+            base_unet_state = copy.deepcopy(unet_segmenter.model.state_dict())
+        else:
+            print(f"Precomputing U-Net seg maps for {len(records)} samples (once for all folds)...")
+            all_seg_maps = precompute_unet_seg_maps(records, unet_segmenter, args.unet_threshold)
+
+    fold_metrics: List[Dict] = []
+    roc_curves = []
+
+    for fold_idx, (train_idx, val_idx) in enumerate(splits):
+        start = time.time()
+        print(f"\n{'='*60}")
+        print(f"  Fold {fold_idx+1}/{args.n_splits}  "
+              f"(train={len(train_idx)}, val={len(val_idx)})")
+        print(f"{'='*60}")
+
+        fold_dir = out_dir / f"fold{fold_idx}"
+        fold_dir.mkdir(parents=True, exist_ok=True)
+
+        train_recs = [records[i] for i in train_idx]
+        val_recs   = [records[i] for i in val_idx]
+
+        if unet_segmenter is not None and args.finetune_epochs > 0:
+            # Restore base REFUGE weights, then fine-tune on this fold's training data only
+            unet_segmenter.model.load_state_dict(copy.deepcopy(base_unet_state))
+            print(f"  Fine-tuning U-Net for {args.finetune_epochs} epochs on training fold...")
+            finetune_unet(
+                unet_segmenter, train_recs,
+                epochs=args.finetune_epochs,
+                lr=args.finetune_lr,
+                batch_size=args.finetune_batch_size,
+                device=device,
+            )
+            print(f"  Generating seg maps with fine-tuned U-Net...")
+            train_maps = precompute_unet_seg_maps(train_recs, unet_segmenter, args.unet_threshold)
+            val_maps   = precompute_unet_seg_maps(val_recs,   unet_segmenter, args.unet_threshold)
+        else:
+            train_maps = [all_seg_maps[i] for i in train_idx] if all_seg_maps else None
+            val_maps   = [all_seg_maps[i] for i in val_idx]   if all_seg_maps else None
+
+        auc, acc, y_true, y_prob, fpr, tpr = _train_one_split(
+            train_recs, val_recs, y[train_idx], args, device,
+            fold_dir, label=f"Fold {fold_idx+1}",
+            train_seg_maps=train_maps, val_seg_maps=val_maps,
+        )
+        elapsed = time.time() - start
+        roc_curves.append((fpr, tpr, auc))
+        print(f"  Fold {fold_idx+1}  AUC={auc:.4f}  ACC={acc:.4f}  ({elapsed:.0f}s)")
+
+        pd.DataFrame({"y_true": y_true, "y_prob": y_prob}).to_csv(
+            fold_dir / "val_probs.csv", index=False
+        )
+        fold_metrics.append({
+            "fold": fold_idx, "auc": auc, "acc": acc,
+            "n_train": len(train_idx), "n_val": len(val_idx), "elapsed_s": elapsed,
+        })
+
+    metrics_df = pd.DataFrame(fold_metrics)
+    metrics_df.to_csv(out_dir / "fold_metrics.csv", index=False)
+
+    mean_auc = float(metrics_df["auc"].mean())
+    std_auc  = float(metrics_df["auc"].std())
+    mean_acc = float(metrics_df["acc"].mean())
+    std_acc  = float(metrics_df["acc"].std())
+
+    print(f"\n{'='*60}")
+    print(f"  CV Summary ({args.n_splits} folds)")
+    print(f"  AUC = {mean_auc:.4f} ± {std_auc:.4f}")
+    print(f"  ACC = {mean_acc:.4f} ± {std_acc:.4f}")
+    print(f"{'='*60}\n")
+
+    pd.DataFrame([{
+        "backbone": args.backbone, "seg_mode": args.seg_mode,
+        "channels": args.channels, "pretrained": not args.no_pretrained,
+        "epochs": args.epochs, "lr": args.lr, "dropout": args.dropout,
+        "auc_mean": mean_auc, "auc_std": std_auc,
+        "acc_mean": mean_acc, "acc_std": std_acc,
+        "n_folds": args.n_splits,
+    }]).to_csv(out_dir / "summary.csv", index=False)
+
+    # Mean ROC curve
+    mean_fpr = np.linspace(0, 1, 200)
+    tprs = [np.interp(mean_fpr, fpr, tpr) for fpr, tpr, _ in roc_curves]
+    mean_tpr = np.mean(tprs, axis=0); mean_tpr[-1] = 1.0
+    std_tpr  = np.std(tprs, axis=0)
+    fig, ax = plt.subplots(figsize=(5.5, 4.5))
+    ax.plot(mean_fpr, mean_tpr, lw=2,
+            label=f"AUC = {mean_auc:.3f} ± {std_auc:.3f}")
+    ax.fill_between(mean_fpr,
+                    np.maximum(mean_tpr - std_tpr, 0),
+                    np.minimum(mean_tpr + std_tpr, 1),
+                    alpha=0.2, color="steelblue")
+    ax.plot([0, 1], [0, 1], "k--", lw=1)
+    ax.set_xlabel("False Positive Rate"); ax.set_ylabel("True Positive Rate")
+    ax.set_title(f"Seg-map CNN ({args.backbone}, {args.seg_mode})")
+    ax.legend(loc="lower right"); ax.grid(True, alpha=0.3, linestyle="--")
+    fig.tight_layout(); fig.savefig(out_dir / "roc_mean.png", dpi=170); plt.close(fig)
+
+    return metrics_df
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        description="Train a CNN on optic disc/cup segmentation maps.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    # --- Data paths ---
+    p.add_argument("--image-dir",    required=True,  help="PAPILA FundusImages directory")
+    p.add_argument("--clinical-dir", required=True,  help="PAPILA ClinicalData directory")
+    p.add_argument("--manifest",     required=True,  help="manifest.csv with annotation paths")
+    p.add_argument("--output-dir",   default="analysis_data/seg_cnn",
+                   help="Where to save results")
+
+    # --- Segmentation mode ---
+    p.add_argument("--seg-mode", choices=["gt", "unet"], default="gt",
+                   help="gt = expert annotations; unet = predicted masks from trained U-Net")
+    p.add_argument("--unet-weights",   default=None,
+                   help="Path to trained UNet weights (.pt); required for --seg-mode unet")
+    p.add_argument("--unet-normalize", default="per_image",
+                   choices=["none", "per_image", "imagenet"],
+                   help="Normalisation used when the UNet was trained")
+    p.add_argument("--unet-threshold", type=float, default=0.5,
+                   help="Sigmoid threshold for binary mask from U-Net logits")
+    p.add_argument("--seg-size", type=int, default=512,
+                   help="Spatial size at which GT contours are rasterised / U-Net runs")
+
+    # --- Model ---
+    p.add_argument("--backbone", default="resnet18",
+                   choices=["resnet18", "resnet50", "efficientnet_b0"],
+                   help="CNN backbone")
+    p.add_argument("--channels", type=int, default=3, choices=[1, 3],
+                   help="1 = single-channel normalised label map; "
+                        "3 = one-hot [bg, disc_rim, cup]")
+    p.add_argument("--no-pretrained", action="store_true",
+                   help="Do not load ImageNet weights for the backbone")
+    p.add_argument("--dropout", type=float, default=0.3)
+
+    # --- Training ---
+    p.add_argument("--epochs",     type=int,   default=60)
+    p.add_argument("--batch-size", type=int,   default=16)
+    p.add_argument("--lr",         type=float, default=1e-4)
+    p.add_argument("--wd",         type=float, default=1e-4,
+                   help="AdamW weight decay")
+    p.add_argument("--img-size",   type=int,   default=224,
+                   help="CNN input spatial resolution")
+    p.add_argument("--no-crop", action="store_true",
+                   help="Disable disc-region cropping (keeps full-image seg map)")
+    p.add_argument("--workers",    type=int,   default=4,
+                   help="DataLoader num_workers")
+
+    # --- U-Net fine-tuning (only applies with --seg-mode unet) ---
+    p.add_argument("--finetune-epochs",     type=int,   default=0,
+                   help="Epochs to fine-tune U-Net on each fold's training data "
+                        "(0 = disabled, uses base REFUGE weights as-is)")
+    p.add_argument("--finetune-lr",         type=float, default=1e-5,
+                   help="Learning rate for U-Net fine-tuning")
+    p.add_argument("--finetune-batch-size", type=int,   default=4,
+                   help="Batch size for U-Net fine-tuning")
+
+    # --- CV ---
+    p.add_argument("--n-splits", type=int, default=5, help="Number of CV folds")
+    p.add_argument("--seed",     type=int, default=42)
+
+    # --- Misc ---
+    p.add_argument("--device", default=None,
+                   help="torch device string (default: cuda if available)")
+    p.add_argument("--label-col", default="Diagnosis",
+                   help="Label column in PAPILA clinical xlsx")
+
+    return p
+
+
+def main(argv=None):
+    args = build_parser().parse_args(argv)
+
+    torch.manual_seed(args.seed)
+    np.random.seed(args.seed)
+
+    device = torch.device(
+        args.device if args.device
+        else ("cuda" if torch.cuda.is_available() else "cpu")
+    )
+    print(f"Device: {device}")
+
+    # ---- Load PAPILA labels ----
+    clinical_df = load_papila_labels(
+        Path(args.clinical_dir),
+        label_col=args.label_col,
+        drop_suspects=True,
+    )
+    print(f"Clinical labels loaded: {len(clinical_df)} eye records "
+          f"(N={int((clinical_df.label==0).sum())}, G={int((clinical_df.label==1).sum())})")
+
+    # ---- Build records ----
+    records = build_records(Path(args.manifest), clinical_df)
+    print(f"Matched records: {len(records)} "
+          f"(N={sum(r.label==0 for r in records)}, G={sum(r.label==1 for r in records)})")
+
+    if not records:
+        print("ERROR: No records matched.  Check manifest and clinical data paths.")
+        sys.exit(1)
+
+    # ---- Load UNet if needed ----
+    unet_segmenter = None
+    if args.seg_mode == "unet":
+        if not args.unet_weights:
+            print("ERROR: --seg-mode unet requires --unet-weights")
+            sys.exit(1)
+        from v3.classes.unet_segmenter import UNetSegmenter  # noqa
+        import tempfile, csv, os
+
+        # Build a minimal manifest for UNetSegmenter init
+        fd, tmp = tempfile.mkstemp(suffix=".csv")
+        with os.fdopen(fd, "w", newline="") as f:
+            writer = csv.writer(f)
+            writer.writerow([
+                "sample_id", "dataset", "image_path",
+                "annotation_disc", "annotation_cup",
+                "annotation_type_disc", "annotation_type_cup", "split",
+            ])
+            r = records[0]
+            writer.writerow([
+                r.sample_id, "papila", str(r.image_path),
+                str(r.annotation_disc), str(r.annotation_cup),
+                r.annotation_type_disc, r.annotation_type_cup, "train",
+            ])
+        unet_segmenter = UNetSegmenter(
+            manifest_path=Path(tmp),
+            normalize=args.unet_normalize,
+            target_size=args.seg_size,
+        )
+        os.unlink(tmp)
+        state = torch.load(args.unet_weights, map_location=unet_segmenter.device)
+        state_dict = state.get("model", state)
+        unet_segmenter.model.load_state_dict(state_dict)
+        unet_segmenter.model.to(unet_segmenter.device)
+        unet_segmenter.model.eval()
+        print(f"U-Net weights loaded from {args.unet_weights}")
+
+    # ---- Run CV ----
+    out_dir = Path(args.output_dir)
+    run_cv(
+        records=records,
+        args=args,
+        device=device,
+        unet_segmenter=unet_segmenter,
+        out_dir=out_dir,
+    )
+
+
+if __name__ == "__main__":
+    main()

v3/scripts/main/phase2/phase_2_overnight.sh

@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+# Phase 2 overnight batch — image-only ResNet50, 10x5 rep-CV
+# Runs 6 configurations:
+#   1. Leaky CV (eye-level splits)
+#   2. Proper CV (patient-level, baseline)
+#   3. GT crop  scale=1.1  (paper-matched tight crop)
+#   4. GT crop  scale=2.5  (default generous crop)
+#   5. UNet crop scale=1.1
+#   6. UNet crop scale=2.5
+set -euo pipefail
+
+SCRIPT="python -m v3.scripts.main.run_cv"
+OUTROOT="v3/results/phase2"
+MANIFEST="manifest.csv"
+UNET_WEIGHTS="models/v2/refuge/segmentation/per_image/best.pt"
+
+BASE="--eval-mode binary \
+      --tower-mode single \
+      --bridge-mode image_only \
+      --backbone resnet50 \
+      --epochs 30 \
+      --augment \
+      --in-memory-cache \
+      --reps 10 \
+      --rep-seed-start 100 \
+      --rep-seed-step 100 \
+      --output-root ${OUTROOT}"
+
+echo "============================================================"
+echo " Phase 2 overnight batch"
+echo " $(date)"
+echo "============================================================"
+
+# ----------------------------------------------------------------
+# 1. Leaky CV (eye-level splits, no crop)
+# ----------------------------------------------------------------
+echo ""
+echo "=== [1/6] Leaky CV (eye-level) ==="
+$SCRIPT $BASE \
+    --leaky-cv \
+    --run-name imageonly_resnet50_leaky
+
+# ----------------------------------------------------------------
+# 2. Proper CV (patient-level, no crop) — baseline
+# ----------------------------------------------------------------
+echo ""
+echo "=== [2/6] Proper CV (patient-level, baseline) ==="
+$SCRIPT $BASE \
+    --run-name imageonly_resnet50_proper
+
+# ----------------------------------------------------------------
+# 3. GT crop, scale=1.1  (paper-matched tight crop)
+# ----------------------------------------------------------------
+echo ""
+echo "=== [3/6] GT crop, scale=1.1 ==="
+$SCRIPT $BASE \
+    --img-crop-gt \
+    --img-crop-manifest ${MANIFEST} \
+    --img-crop-scale 1.1 \
+    --img-crop-size 200 \
+    --run-name imageonly_resnet50_gtcrop_1.1
+
+# ----------------------------------------------------------------
+# 4. GT crop, scale=2.5  (default generous crop)
+# ----------------------------------------------------------------
+echo ""
+echo "=== [4/6] GT crop, scale=2.5 ==="
+$SCRIPT $BASE \
+    --img-crop-gt \
+    --img-crop-manifest ${MANIFEST} \
+    --img-crop-scale 2.5 \
+    --img-crop-size 200 \
+    --run-name imageonly_resnet50_gtcrop_2.5
+
+# ----------------------------------------------------------------
+# 5. UNet crop, scale=1.1
+# ----------------------------------------------------------------
+echo ""
+echo "=== [5/6] UNet crop, scale=1.1 ==="
+$SCRIPT $BASE \
+    --img-crop-weights ${UNET_WEIGHTS} \
+    --img-crop-manifest ${MANIFEST} \
+    --img-crop-scale 1.1 \
+    --img-crop-size 200 \
+    --run-name imageonly_resnet50_unetcrop_1.1
+
+# ----------------------------------------------------------------
+# 6. UNet crop, scale=2.5
+# ----------------------------------------------------------------
+echo ""
+echo "=== [6/6] UNet crop, scale=2.5 ==="
+$SCRIPT $BASE \
+    --img-crop-weights ${UNET_WEIGHTS} \
+    --img-crop-manifest ${MANIFEST} \
+    --img-crop-scale 2.5 \
+    --img-crop-size 200 \
+    --run-name imageonly_resnet50_unetcrop_2.5
+
+# ----------------------------------------------------------------
+# 7. Refugelike backbone (proper CV, no crop) — pre-training effect
+# ----------------------------------------------------------------
+echo ""
+echo "=== [7/7] Refugelike backbone (proper CV, no crop) ==="
+$SCRIPT $BASE \
+    --backbone refugelike \
+    --run-name imageonly_refugelike_proper
+
+echo ""
+echo "============================================================"
+echo " All done — $(date)"
+echo "============================================================"

v3/scripts/main/phase3/dispatch_phase3.py

@@ -0,0 +1,197 @@
+"""
+Dispatch phase 3 experiment runs to the distributed job server.
+
+Reads experiment_grid.json, checks which runs already have complete 10x5 results,
+and submits the rest via submit-cv. Skips runs marked needs_implementation.
+
+Usage:
+    python -m v3.scripts.main.phase3.dispatch_phase3 \
+        --server http://hades:8765 --token hypertower
+
+    # Dry run (print what would be submitted, don't actually submit):
+    python -m v3.scripts.main.phase3.dispatch_phase3 \
+        --server http://hades:8765 --token hypertower --dry-run
+
+    # Override number of reps (default 10):
+    python -m v3.scripts.main.phase3.dispatch_phase3 \
+        --server http://hades:8765 --token hypertower --reps 4
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+import requests
+
+# Allow running as `python v3/scripts/main/phase3/dispatch_phase3.py`
+sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
+
+GRID_PATH   = Path(__file__).parent / "experiment_grid.json"
+RESULTS_ROOT = Path(__file__).resolve().parents[4] / "v3" / "results"
+MODULE      = "v3.scripts.main.run_cv"
+OUTPUT_DIR  = "v3/results"
+REP_SEED_START = 100
+REP_SEED_STEP  = 100
+
+
+# ── Completion check ──────────────────────────────────────────────────────────
+
+def _completed_reps(run_name: str, reps: int) -> list[int]:
+    """Return list of rep indices that already have a summary.json."""
+    done = []
+    for i in range(reps):
+        summary = RESULTS_ROOT / run_name / f"rep{i:02d}" / "binary" / "single" / "summary.json"
+        if summary.exists():
+            done.append(i)
+    return done
+
+
+# ── Server API ────────────────────────────────────────────────────────────────
+
+class _API:
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self._h = {"x-token": token}
+
+    def get(self, path: str, **params) -> object:
+        r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+    def post(self, path: str, body: dict) -> dict:
+        r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+
+def _queued_reps(jobs: list[dict], run_name: str) -> set[int]:
+    """Return rep indices already pending or running in the server queue."""
+    active = set()
+    for job in jobs:
+        if job["run_name"] != run_name:
+            continue
+        if job["state"] not in ("pending", "running"):
+            continue
+        # Extract --rep-index from job args
+        try:
+            args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"])
+            if "--rep-index" in args:
+                active.add(int(args[args.index("--rep-index") + 1]))
+        except Exception:
+            pass
+    return active
+
+
+def _submit_cv(api: _API, run_name: str, run_args: list[str],
+               reps: int, missing: list[int], dry_run: bool):
+    """Submit one job per missing rep."""
+    for i in missing:
+        seed = REP_SEED_START + i * REP_SEED_STEP
+        rep_args = run_args + [
+            "--run-name", run_name,
+            "--reps", "1",
+            "--rep-seed-start", str(seed),
+            "--rep-index", str(i),
+        ]
+        body = {
+            "run_name": run_name,
+            "module": MODULE,
+            "args": rep_args,
+            "output_dir": OUTPUT_DIR,
+            "priority": 0,
+        }
+        if dry_run:
+            print(f"  [dry-run] would queue rep{i:02d}  seed={seed}")
+        else:
+            resp = api.post("/jobs", body)
+            print(f"  queued rep{i:02d}  seed={seed}  job_id={resp['job_id']}")
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                  formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""),
+                    help="Server URL (or set HT_SERVER)")
+    ap.add_argument("--token",  default=os.environ.get("HT_TOKEN",  ""),
+                    help="Shared secret (or set HT_TOKEN)")
+    ap.add_argument("--reps",   type=int, default=10,
+                    help="Expected number of reps per run (default: 10)")
+    ap.add_argument("--grid",   type=Path, default=GRID_PATH,
+                    help="Path to experiment grid JSON (default: experiment_grid.json)")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="Print what would be submitted without actually submitting")
+    args = ap.parse_args()
+
+    if not args.dry_run:
+        if not args.server:
+            ap.error("--server is required (or set HT_SERVER)")
+        if not args.token:
+            ap.error("--token is required (or set HT_TOKEN)")
+    elif not args.server or not args.token:
+        print("[dry-run] no --server/--token provided — skipping queue check, showing disk state only")
+
+    grid = json.loads(args.grid.read_text())
+    common_args = grid["common_args"]
+    api = _API(args.server, args.token) if (args.server and args.token) else None
+
+    # Fetch current server queue once (pending + running)
+    server_jobs: list[dict] = []
+    if api:
+        try:
+            all_jobs = api.get("/jobs")
+            server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")]
+            print(f"[server] {len(server_jobs)} job(s) currently pending/running in queue")
+        except Exception as e:
+            print(f"[warn] could not fetch server queue: {e}")
+
+    # Collect all runs: baseline + every group's runs
+    all_runs = [grid["baseline"]]
+    for group in grid["groups"]:
+        if group.get("needs_implementation"):
+            print(f"\n[skip] group '{group['name']}' — {group['needs_implementation']}")
+            continue
+        all_runs.extend(group["runs"])
+
+    submitted_total = 0
+    skipped_total   = 0
+
+    for run in all_runs:
+        run_name  = run["run_name"]
+        run_args  = common_args + run.get("extra_args", [])
+        done      = set(_completed_reps(run_name, args.reps))
+        queued    = _queued_reps(server_jobs, run_name)
+        accounted = done | queued
+        missing   = [i for i in range(args.reps) if i not in accounted]
+
+        if not missing:
+            if len(done) == args.reps:
+                print(f"\n[done]  {run_name}  ({args.reps}/{args.reps} reps complete)")
+            else:
+                in_q = sorted(queued - done)
+                print(f"\n[skip]  {run_name}  ({len(done)} done, {len(in_q)} queued: {[f'rep{i:02d}' for i in in_q]})")
+            skipped_total += 1
+            continue
+
+        parts = []
+        if done:    parts.append(f"{len(done)} done")
+        if queued:  parts.append(f"{len(queued - done)} queued")
+        status = ", ".join(parts) if parts else "not started"
+        print(f"\n[queue] {run_name}  ({status}) — submitting {len(missing)} rep(s)")
+        _submit_cv(api, run_name, run_args, args.reps, missing, args.dry_run)
+        submitted_total += len(missing)
+
+    print(f"\n{'='*50}")
+    print(f"Submitted: {submitted_total} jobs  |  Already accounted for: {skipped_total} runs")
+    if grid.get("groups"):
+        needs_impl = sum(1 for g in grid["groups"] if g.get("needs_implementation"))
+        if needs_impl:
+            print(f"Skipped (needs implementation): {needs_impl} group(s)")
+
+
+if __name__ == "__main__":
+    main()

v3/scripts/main/phase3/epoch_grid.json

@@ -0,0 +1,66 @@
+{
+  "_notes": [
+    "Epoch length sensitivity experiments.",
+    "All other settings match the phase3 baseline (fused bridge, BCD p=0.5, refugelike, etc.).",
+    "common_args are prepended to every run's args list."
+  ],
+
+  "common_args": [
+    "--eval-mode", "binary",
+    "--tower-mode", "single",
+    "--in-memory-cache",
+    "--augment",
+    "--tune-binary-threshold",
+    "--backbone", "refugelike",
+    "--output-root", "v3/results"
+  ],
+
+  "_common_args_implicit_defaults": {
+    "--bridge-mode": "fused",
+    "--tower-loss-mode": "bcd",
+    "--bcd-prob": "0.5",
+    "--warmup-cd-epochs": "40",
+    "--single-warmup-tower-epochs": "3",
+    "--single-warmup-fused-epochs": "3"
+  },
+
+  "baseline": {
+    "run_name": "phase3/epochs_30",
+    "description": "30-epoch run — same as phase3 baseline, included here for direct comparison.",
+    "extra_args": ["--epochs", "30"]
+  },
+
+  "groups": [
+    {
+      "name": "epoch_length",
+      "description": "Test sensitivity to total training epochs (warmup epochs unchanged).",
+      "runs": [
+        {
+          "run_name": "phase3/epochs_1",
+          "description": "1 epoch total — effectively pure warmup output with a single main-phase step.",
+          "extra_args": ["--epochs", "1"]
+        },
+        {
+          "run_name": "phase3/epochs_5",
+          "description": "5 epochs total.",
+          "extra_args": ["--epochs", "5"]
+        },
+        {
+          "run_name": "phase3/epochs_10",
+          "description": "10 epochs total.",
+          "extra_args": ["--epochs", "10"]
+        },
+        {
+          "run_name": "phase3/epochs_20",
+          "description": "20 epochs total.",
+          "extra_args": ["--epochs", "20"]
+        },
+        {
+          "run_name": "phase3/epochs_50",
+          "description": "50 epochs total.",
+          "extra_args": ["--epochs", "50"]
+        }
+      ]
+    }
+  ]
+}

v3/scripts/main/phase3/experiment_grid.json

@@ -0,0 +1,362 @@
+{
+  "_notes": [
+    "All runs use single-eye tower mode, binary eval, 10x5 rep-CV.",
+    "common_args are prepended to every run's args list.",
+    "Entries marked 'needs_implementation' require small code changes before running (noted inline).",
+    "Bridge fusion uses elementwise product of projected image/clinical features.",
+    "SE infrastructure exists in Bridge/ImageTower/ClinicalTower but use_se is hardcoded False",
+    "  in SingleEyeHT \u2014 add --se-img-tower / --se-cd-tower / --se-bridge flags to wire through.",
+    "Dropout is hardcoded: Bridge classifier=0.5, ClinicalTower=0.1 \u2014 add --bridge-dropout /",
+    "  --cd-dropout flags to make configurable."
+  ],
+  "common_args": [
+    "--eval-mode",
+    "binary",
+    "--tower-mode",
+    "single",
+    "--epochs",
+    "30",
+    "--in-memory-cache",
+    "--augment",
+    "--tune-binary-threshold",
+    "--backbone",
+    "refugelike",
+    "--output-root",
+    "v3/results"
+  ],
+  "_common_args_implicit_defaults": {
+    "--bridge-mode": "fused",
+    "--tower-loss-mode": "bcd",
+    "--bcd-prob": "0.5",
+    "--warmup-cd-epochs": "40",
+    "--single-warmup-tower-epochs": "3",
+    "--single-warmup-fused-epochs": "3"
+  },
+  "baseline": {
+    "run_name": "phase3/baseline",
+    "description": "Fused bridge (image+clinical), BCD p=0.5, cd_warmup=40, tower/fused warmup=3/3, no SE, no IOP correction.",
+    "extra_args": []
+  },
+  "groups": [
+    {
+      "name": "loss_function",
+      "description": "Test BCD loss variants vs cross-entropy baseline.",
+      "runs": [
+        {
+          "run_name": "phase3/loss_all",
+          "description": "All-losses mode (cross-entropy on all three heads every step).",
+          "extra_args": [
+            "--tower-loss-mode",
+            "all"
+          ]
+        },
+        {
+          "run_name": "phase3/loss_bcd_p03",
+          "description": "BCD with lower switching probability (more CE, less BCD).",
+          "extra_args": [
+            "--tower-loss-mode",
+            "bcd",
+            "--bcd-prob",
+            "0.3"
+          ]
+        },
+        {
+          "run_name": "phase3/loss_bcd_p07",
+          "description": "BCD with higher switching probability (more BCD, less CE).",
+          "extra_args": [
+            "--tower-loss-mode",
+            "bcd",
+            "--bcd-prob",
+            "0.7"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "se_attention",
+      "description": "Squeeze-and-excitation gates at different points in the network.",
+      "runs": [
+        {
+          "run_name": "phase3/se_bridge",
+          "description": "SE gate on fused vector inside the bridge only.",
+          "extra_args": [
+            "--se-bridge"
+          ]
+        },
+        {
+          "run_name": "phase3/se_img_tower",
+          "description": "SE gate on image tower output features.",
+          "extra_args": [
+            "--se-img-tower"
+          ]
+        },
+        {
+          "run_name": "phase3/se_cd_tower",
+          "description": "SE gate on clinical tower output features.",
+          "extra_args": [
+            "--se-cd-tower"
+          ]
+        },
+        {
+          "run_name": "phase3/se_all",
+          "description": "SE gates on image tower, clinical tower, and bridge.",
+          "extra_args": [
+            "--se-img-tower",
+            "--se-cd-tower",
+            "--se-bridge"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "iop_correction",
+      "description": "Test different IOP measurement correction strategies (default: no correction).",
+      "runs": [
+        {
+          "run_name": "phase3/iop_ratio",
+          "description": "IOP correction via Perkins\u2192Pneumatic ratio scaling.",
+          "extra_args": [
+            "--iop-corr-method",
+            "ratio"
+          ]
+        },
+        {
+          "run_name": "phase3/iop_ols",
+          "description": "IOP correction via OLS regression.",
+          "extra_args": [
+            "--iop-corr-method",
+            "ols"
+          ]
+        },
+        {
+          "run_name": "phase3/iop_lad",
+          "description": "IOP correction via LAD (robust to outliers) regression.",
+          "extra_args": [
+            "--iop-corr-method",
+            "lad"
+          ]
+        },
+        {
+          "run_name": "phase3/iop_multi",
+          "description": "IOP correction via multivariate regression including pachymetry.",
+          "extra_args": [
+            "--iop-corr-method",
+            "multi"
+          ]
+        },
+        {
+          "run_name": "phase3/iop_ratio_drop_raw",
+          "description": "Ratio correction + drop raw IOP (only corrected IOP seen by model).",
+          "extra_args": [
+            "--iop-corr-method",
+            "ratio",
+            "--iop-drop-raw"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "feature_ablation",
+      "description": "Exclude individual clinical features to measure each one's contribution.",
+      "runs": [
+        {
+          "run_name": "phase3/excl_iop",
+          "description": "No IOP features \u2014 tests how much intraocular pressure contributes.",
+          "extra_args": [
+            "--exclude-cols",
+            "IOP",
+            "Pachymetry"
+          ]
+        },
+        {
+          "run_name": "phase3/excl_age",
+          "description": "No age feature.",
+          "extra_args": [
+            "--exclude-cols",
+            "Age"
+          ]
+        },
+        {
+          "run_name": "phase3/excl_axial_length",
+          "description": "No axial length feature.",
+          "extra_args": [
+            "--exclude-cols",
+            "Axial_Length"
+          ]
+        },
+        {
+          "run_name": "phase3/excl_refractive",
+          "description": "No refractive defect feature.",
+          "extra_args": [
+            "--exclude-cols",
+            "Refractive_Defect"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "network_dims",
+      "description": "Test sensitivity to clinical tower and bridge fusion dimensionality.",
+      "runs": [
+        {
+          "run_name": "phase3/cd_hidden_64",
+          "description": "Smaller clinical tower (64 hidden units vs default 128).",
+          "extra_args": [
+            "--cd-hidden-dim",
+            "64"
+          ]
+        },
+        {
+          "run_name": "phase3/cd_hidden_256",
+          "description": "Larger clinical tower (256 hidden units vs default 128).",
+          "extra_args": [
+            "--cd-hidden-dim",
+            "256"
+          ]
+        },
+        {
+          "run_name": "phase3/fusion_dim_128",
+          "description": "Smaller fusion space (128 vs default 256).",
+          "extra_args": [
+            "--fusion-dim",
+            "128"
+          ]
+        },
+        {
+          "run_name": "phase3/fusion_dim_512",
+          "description": "Larger fusion space (512 vs default 256).",
+          "extra_args": [
+            "--fusion-dim",
+            "512"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "backbone_freezing",
+      "description": "Partial backbone freezing to reduce overfitting and speed training.",
+      "runs": [
+        {
+          "run_name": "phase3/freeze_25",
+          "description": "Freeze earliest 25% of backbone blocks.",
+          "extra_args": [
+            "--freeze-ratio",
+            "0.25"
+          ]
+        },
+        {
+          "run_name": "phase3/freeze_50",
+          "description": "Freeze earliest 50% of backbone blocks.",
+          "extra_args": [
+            "--freeze-ratio",
+            "0.50"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "learning_rate",
+      "description": "Test LR sensitivity (default 1e-4).",
+      "runs": [
+        {
+          "run_name": "phase3/lr_1e3",
+          "description": "Higher learning rate 1e-3.",
+          "extra_args": [
+            "--lr",
+            "1e-3"
+          ]
+        },
+        {
+          "run_name": "phase3/lr_3e4",
+          "description": "Intermediate learning rate 3e-4.",
+          "extra_args": [
+            "--lr",
+            "3e-4"
+          ]
+        },
+        {
+          "run_name": "phase3/lr_1e5",
+          "description": "Lower learning rate 1e-5.",
+          "extra_args": [
+            "--lr",
+            "1e-5"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "dropout",
+      "description": "Test bridge classifier and clinical tower dropout rates.",
+      "runs": [
+        {
+          "run_name": "phase3/bridge_dropout_03",
+          "description": "Reduce bridge classifier dropout from 0.5 to 0.3.",
+          "extra_args": [
+            "--bridge-dropout",
+            "0.3"
+          ]
+        },
+        {
+          "run_name": "phase3/bridge_dropout_07",
+          "description": "Increase bridge classifier dropout to 0.7.",
+          "extra_args": [
+            "--bridge-dropout",
+            "0.7"
+          ]
+        },
+        {
+          "run_name": "phase3/cd_dropout_03",
+          "description": "Increase clinical tower dropout from 0.1 to 0.3.",
+          "extra_args": [
+            "--cd-dropout",
+            "0.3"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "warmup",
+      "description": "Test warmup ablations vs default (cd=40, tower/fused=3/3).",
+      "runs": [
+        {
+          "run_name": "phase3/warmup_no_cd",
+          "description": "No CD warmup (cd=0) \u2014 tests whether the 40-epoch CD warmup is necessary.",
+          "extra_args": [
+            "--warmup-cd-epochs",
+            "0"
+          ]
+        },
+        {
+          "run_name": "phase3/warmup_tower5_fused5",
+          "description": "Extended tower/fused warmup (5/5 vs default 3/3).",
+          "extra_args": [
+            "--single-warmup-tower-epochs",
+            "5",
+            "--single-warmup-fused-epochs",
+            "5"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "sampling_augmentation",
+      "description": "Test data sampling and augmentation choices.",
+      "runs": [
+        {
+          "run_name": "phase3/no_augment",
+          "description": "No augmentation \u2014 baseline images only.",
+          "extra_args": [
+            "--no-augment"
+          ]
+        },
+        {
+          "run_name": "phase3/balanced_sampling",
+          "description": "Weighted balanced sampler to counter class imbalance.",
+          "extra_args": [
+            "--balanced-sampling"
+          ]
+        }
+      ]
+    }
+  ]
+}

v3/scripts/main/phase3/phase35_grid.json

@@ -0,0 +1,54 @@
+{
+  "_notes": [
+    "Phase 3.5 — confirmation and BCD tuning.",
+    "All runs use best settings from phase 3: refugelike backbone, ratio IOP correction, drop raw IOP, exclude axial length.",
+    "Baseline here is phase3/iop_ratio_drop_raw (0.8685 ± 0.011) — already complete, not re-run.",
+    "common_args are prepended to every run's args list."
+  ],
+
+  "common_args": [
+    "--eval-mode", "binary",
+    "--tower-mode", "single",
+    "--epochs", "30",
+    "--in-memory-cache",
+    "--augment",
+    "--tune-binary-threshold",
+    "--backbone", "refugelike",
+    "--iop-corr-method", "ratio",
+    "--iop-drop-raw",
+    "--exclude-cols", "Axial_Length",
+    "--output-root", "v3/results"
+  ],
+
+  "_common_args_implicit_defaults": {
+    "--bridge-mode": "fused",
+    "--warmup-cd-epochs": "40",
+    "--single-warmup-tower-epochs": "3",
+    "--single-warmup-fused-epochs": "3"
+  },
+
+  "baseline": {
+    "run_name": "phase35/iop_bcd_p07",
+    "description": "Best IOP preprocessing + best BCD prob from phase 3 combined.",
+    "extra_args": ["--tower-loss-mode", "bcd", "--bcd-prob", "0.7"]
+  },
+
+  "groups": [
+    {
+      "name": "bcd_tuning",
+      "description": "Extended BCD probability sweep with best IOP settings.",
+      "runs": [
+        {
+          "run_name": "phase35/iop_bcd_p08",
+          "description": "BCD p=0.8 with ratio IOP + drop raw.",
+          "extra_args": ["--tower-loss-mode", "bcd", "--bcd-prob", "0.8"]
+        },
+        {
+          "run_name": "phase35/iop_bcd_p09",
+          "description": "BCD p=0.9 with ratio IOP + drop raw.",
+          "extra_args": ["--tower-loss-mode", "bcd", "--bcd-prob", "0.9"]
+        }
+      ]
+    }
+  ]
+}

v3/scripts/main/phase4/dispatch_phase4.py

@@ -0,0 +1,199 @@
+"""
+Dispatch phase 4 experiment runs to the distributed job server.
+
+Reads experiment_grid.json, checks which runs already have complete 10x5 results,
+and submits the rest. Skips runs marked needs_implementation.
+
+Usage:
+    python -m v3.scripts.main.phase4.dispatch_phase4 \
+        --server http://hades:8765 --token hypertower
+
+    # Dry run (print what would be submitted, don't actually submit):
+    python -m v3.scripts.main.phase4.dispatch_phase4 \
+        --server http://hades:8765 --token hypertower --dry-run
+
+    # Override number of reps (default 10):
+    python -m v3.scripts.main.phase4.dispatch_phase4 \
+        --server http://hades:8765 --token hypertower --reps 4
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+import requests
+
+# Allow running as `python v3/scripts/main/phase3/dispatch_phase3.py`
+sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
+
+GRID_PATH   = Path(__file__).parent / "experiment_grid.json"
+RESULTS_ROOT = Path(__file__).resolve().parents[4] / "v3" / "results"
+MODULE      = "v3.scripts.main.run_cv"
+OUTPUT_DIR  = "v3/results"
+REP_SEED_START = 100
+REP_SEED_STEP  = 100
+
+
+# ── Completion check ──────────────────────────────────────────────────────────
+
+def _completed_reps(run_name: str, reps: int) -> list[int]:
+    """Return list of rep indices that already have a summary.json (any tower mode)."""
+    done = []
+    for i in range(reps):
+        rep_dir = RESULTS_ROOT / run_name / f"rep{i:02d}" / "binary"
+        # Accept any tower mode subdir
+        if rep_dir.exists() and any((rep_dir / tm / "summary.json").exists()
+                                     for tm in ("single", "bilateral", "siamese", "ensemble")):
+            done.append(i)
+    return done
+
+
+# ── Server API ────────────────────────────────────────────────────────────────
+
+class _API:
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self._h = {"x-token": token}
+
+    def get(self, path: str, **params) -> object:
+        r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+    def post(self, path: str, body: dict) -> dict:
+        r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+
+def _queued_reps(jobs: list[dict], run_name: str) -> set[int]:
+    """Return rep indices already pending or running in the server queue."""
+    active = set()
+    for job in jobs:
+        if job["run_name"] != run_name:
+            continue
+        if job["state"] not in ("pending", "running"):
+            continue
+        # Extract --rep-index from job args
+        try:
+            args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"])
+            if "--rep-index" in args:
+                active.add(int(args[args.index("--rep-index") + 1]))
+        except Exception:
+            pass
+    return active
+
+
+def _submit_cv(api: _API, run_name: str, run_args: list[str],
+               reps: int, missing: list[int], dry_run: bool):
+    """Submit one job per missing rep."""
+    for i in missing:
+        seed = REP_SEED_START + i * REP_SEED_STEP
+        rep_args = run_args + [
+            "--run-name", run_name,
+            "--reps", "1",
+            "--rep-seed-start", str(seed),
+            "--rep-index", str(i),
+        ]
+        body = {
+            "run_name": run_name,
+            "module": MODULE,
+            "args": rep_args,
+            "output_dir": OUTPUT_DIR,
+            "priority": 0,
+        }
+        if dry_run:
+            print(f"  [dry-run] would queue rep{i:02d}  seed={seed}")
+        else:
+            resp = api.post("/jobs", body)
+            print(f"  queued rep{i:02d}  seed={seed}  job_id={resp['job_id']}")
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                  formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""),
+                    help="Server URL (or set HT_SERVER)")
+    ap.add_argument("--token",  default=os.environ.get("HT_TOKEN",  ""),
+                    help="Shared secret (or set HT_TOKEN)")
+    ap.add_argument("--reps",   type=int, default=10,
+                    help="Expected number of reps per run (default: 10)")
+    ap.add_argument("--grid",   type=Path, default=GRID_PATH,
+                    help="Path to experiment grid JSON (default: experiment_grid.json)")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="Print what would be submitted without actually submitting")
+    args = ap.parse_args()
+
+    if not args.dry_run:
+        if not args.server:
+            ap.error("--server is required (or set HT_SERVER)")
+        if not args.token:
+            ap.error("--token is required (or set HT_TOKEN)")
+    elif not args.server or not args.token:
+        print("[dry-run] no --server/--token provided — skipping queue check, showing disk state only")
+
+    grid = json.loads(args.grid.read_text())
+    common_args = grid["common_args"]
+    api = _API(args.server, args.token) if (args.server and args.token) else None
+
+    # Fetch current server queue once (pending + running)
+    server_jobs: list[dict] = []
+    if api:
+        try:
+            all_jobs = api.get("/jobs")
+            server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")]
+            print(f"[server] {len(server_jobs)} job(s) currently pending/running in queue")
+        except Exception as e:
+            print(f"[warn] could not fetch server queue: {e}")
+
+    # Collect all runs: baseline + every group's runs
+    all_runs = [grid["baseline"]]
+    for group in grid["groups"]:
+        if group.get("needs_implementation"):
+            print(f"\n[skip] group '{group['name']}' — {group['needs_implementation']}")
+            continue
+        all_runs.extend(group["runs"])
+
+    submitted_total = 0
+    skipped_total   = 0
+
+    for run in all_runs:
+        run_name  = run["run_name"]
+        run_args  = common_args + run.get("extra_args", [])
+        done      = set(_completed_reps(run_name, args.reps))
+        queued    = _queued_reps(server_jobs, run_name)
+        accounted = done | queued
+        missing   = [i for i in range(args.reps) if i not in accounted]
+
+        if not missing:
+            if len(done) == args.reps:
+                print(f"\n[done]  {run_name}  ({args.reps}/{args.reps} reps complete)")
+            else:
+                in_q = sorted(queued - done)
+                print(f"\n[skip]  {run_name}  ({len(done)} done, {len(in_q)} queued: {[f'rep{i:02d}' for i in in_q]})")
+            skipped_total += 1
+            continue
+
+        parts = []
+        if done:    parts.append(f"{len(done)} done")
+        if queued:  parts.append(f"{len(queued - done)} queued")
+        status = ", ".join(parts) if parts else "not started"
+        print(f"\n[queue] {run_name}  ({status}) — submitting {len(missing)} rep(s)")
+        _submit_cv(api, run_name, run_args, args.reps, missing, args.dry_run)
+        submitted_total += len(missing)
+
+    print(f"\n{'='*50}")
+    print(f"Submitted: {submitted_total} jobs  |  Already accounted for: {skipped_total} runs")
+    if grid.get("groups"):
+        needs_impl = sum(1 for g in grid["groups"] if g.get("needs_implementation"))
+        if needs_impl:
+            print(f"Skipped (needs implementation): {needs_impl} group(s)")
+
+
+if __name__ == "__main__":
+    main()

v3/scripts/main/phase4/experiment_grid.json

@@ -0,0 +1,77 @@
+{
+  "_notes": [
+    "Phase 4 — Dual CNN architecture comparison (image only).",
+    "Goal: isolate the effect of bilateral processing by comparing architectures without clinical data.",
+    "Best settings from phase 3 carried forward: iop_ratio_drop_raw, bcd_p05 (p07 did not stack).",
+    "common_args are prepended to every run's args list.",
+    "All runs use --bridge-mode image_only — no clinical data."
+  ],
+
+  "common_args": [
+    "--eval-mode", "binary",
+    "--bridge-mode", "image_only",
+    "--epochs", "30",
+    "--in-memory-cache",
+    "--augment",
+    "--tune-binary-threshold",
+    "--backbone", "refugelike",
+    "--iop-corr-method", "ratio",
+    "--iop-drop-raw",
+    "--output-root", "v3/results"
+  ],
+
+  "_common_args_implicit_defaults": {
+    "--tower-loss-mode": "bcd",
+    "--bcd-prob": "0.5",
+    "--warmup-cd-epochs": "0",
+    "--bilat-warmup-tower-epochs": "3",
+    "--bilat-warmup-fused-epochs": "3"
+  },
+
+  "baseline": {
+    "run_name": "phase4/single",
+    "description": "Single-eye baseline with best phase 3 image settings. Direct comparison point for bilateral modes.",
+    "extra_args": ["--tower-mode", "single"]
+  },
+
+  "groups": [
+    {
+      "name": "architecture",
+      "description": "Core bilateral architecture comparison.",
+      "runs": [
+        {
+          "run_name": "phase4/ensemble",
+          "description": "Ensemble: two independent single-eye forward passes, patient-level average of OD+OS scores.",
+          "extra_args": ["--tower-mode", "ensemble"]
+        },
+        {
+          "run_name": "phase4/bilateral",
+          "description": "BilateralHT: shared towers, concat OD+OS → learned joint projection MLP → classifier.",
+          "extra_args": ["--tower-mode", "bilateral"]
+        },
+        {
+          "run_name": "phase4/siamese",
+          "description": "SiameseHT: shared backbone, mean+delta (asymmetry) representation → classifier.",
+          "extra_args": ["--tower-mode", "siamese"]
+        }
+      ]
+    },
+
+    {
+      "name": "loss_bilateral",
+      "description": "Test loss function sensitivity in bilateral modes (using winner from architecture group).",
+      "runs": [
+        {
+          "run_name": "phase4/bilateral_loss_all",
+          "description": "BilateralHT with all-losses mode — joint training dynamics may differ from single-eye.",
+          "extra_args": ["--tower-mode", "bilateral", "--tower-loss-mode", "all"]
+        },
+        {
+          "run_name": "phase4/siamese_loss_all",
+          "description": "SiameseHT with all-losses mode.",
+          "extra_args": ["--tower-mode", "siamese", "--tower-loss-mode", "all"]
+        }
+      ]
+    }
+  ]
+}

v3/scripts/main/phase5/dispatch_logit_mlp_ckpt.py

@@ -0,0 +1,170 @@
+"""
+Dispatch a 10×5 rep-CV of logit_mlp_head with --save-checkpoints.
+
+Results land in v3/results/phase5/logit_mlp_head_ckpt/{rep00..rep09}/binary/ensemble/
+
+Usage:
+    # Dry run
+    python -m v3.scripts.main.phase5.dispatch_logit_mlp_ckpt --dry-run
+
+    # Submit to server
+    python -m v3.scripts.main.phase5.dispatch_logit_mlp_ckpt \
+        --server http://hades:8765 --token hypertower
+
+    # Skip reps already done, re-queue only missing ones:
+    python -m v3.scripts.main.phase5.dispatch_logit_mlp_ckpt \
+        --server http://hades:8765 --token hypertower
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+import requests
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
+
+RUN_NAME       = "phase5/logit_mlp_head_ckpt"
+MODULE         = "v3.scripts.main.run_cv"
+OUTPUT_DIR     = "v3/results"
+RESULTS_ROOT   = Path(__file__).resolve().parents[4] / "v3" / "results"
+REP_SEED_START = 100
+REP_SEED_STEP  = 100
+N_REPS         = 10
+
+RUN_ARGS = [
+    "--eval-mode",            "binary",
+    "--bridge-mode",          "fused",
+    "--tower-mode",           "ensemble",
+    "--fused-head",
+    "--head-type",            "logit_mlp",
+    "--epochs",               "30",
+    "--in-memory-cache",
+    "--augment",
+    "--tune-binary-threshold",
+    "--backbone",             "refugelike",
+    "--iop-corr-method",      "ratio",
+    "--iop-drop-raw",
+    "--exclude-cols",         "Axial_Length",
+    "--output-root",          "v3/results",
+    "--save-checkpoints",
+]
+
+
+# ── Completion check ──────────────────────────────────────────────────────────
+
+def _completed_reps(reps: int) -> list[int]:
+    done = []
+    for i in range(reps):
+        rep_dir = RESULTS_ROOT / RUN_NAME / f"rep{i:02d}" / "binary" / "ensemble"
+        if (rep_dir / "summary.json").exists():
+            # Also verify at least one checkpoint exists
+            if any(rep_dir.glob("fold*/best_single.pt")):
+                done.append(i)
+            else:
+                print(f"  [warn] rep{i:02d} has summary.json but no checkpoints — will re-queue")
+    return done
+
+
+# ── Server API ────────────────────────────────────────────────────────────────
+
+class _API:
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self._h = {"x-token": token}
+
+    def get(self, path: str, **params) -> object:
+        r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+    def post(self, path: str, body: dict) -> dict:
+        r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+
+def _queued_reps(jobs: list[dict]) -> set[int]:
+    active = set()
+    for job in jobs:
+        if job.get("run_name") != RUN_NAME:
+            continue
+        if job["state"] not in ("pending", "running"):
+            continue
+        try:
+            args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"])
+            if "--rep-index" in args:
+                active.add(int(args[args.index("--rep-index") + 1]))
+        except Exception:
+            pass
+    return active
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                  formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--server",  default=os.environ.get("HT_SERVER", ""))
+    ap.add_argument("--token",   default=os.environ.get("HT_TOKEN",  ""))
+    ap.add_argument("--reps",    type=int, default=N_REPS)
+    ap.add_argument("--dry-run", action="store_true")
+    args = ap.parse_args()
+
+    if not args.dry_run:
+        if not args.server:
+            ap.error("--server required (or set HT_SERVER)")
+        if not args.token:
+            ap.error("--token required (or set HT_TOKEN)")
+
+    api = _API(args.server, args.token) if (args.server and args.token) else None
+
+    server_jobs: list[dict] = []
+    if api:
+        try:
+            all_jobs = api.get("/jobs")
+            server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")]
+            print(f"[server] {len(server_jobs)} job(s) pending/running")
+        except Exception as e:
+            print(f"[warn] could not fetch queue: {e}")
+
+    done   = set(_completed_reps(args.reps))
+    queued = _queued_reps(server_jobs)
+    missing = [i for i in range(args.reps) if i not in (done | queued)]
+
+    print(f"\nRun: {RUN_NAME}")
+    print(f"  Done:    {sorted(done)}")
+    print(f"  Queued:  {sorted(queued - done)}")
+    print(f"  Missing: {missing}")
+
+    if not missing:
+        print("Nothing to submit.")
+        return
+
+    for i in missing:
+        seed = REP_SEED_START + i * REP_SEED_STEP
+        rep_args = RUN_ARGS + [
+            "--run-name",       RUN_NAME,
+            "--reps",           "1",
+            "--rep-seed-start", str(seed),
+            "--rep-index",      str(i),
+        ]
+        body = {
+            "run_name":   RUN_NAME,
+            "module":     MODULE,
+            "args":       rep_args,
+            "output_dir": OUTPUT_DIR,
+            "priority":   0,
+        }
+        if args.dry_run:
+            print(f"  [dry-run] rep{i:02d}  seed={seed}")
+        else:
+            resp = api.post("/jobs", body)
+            print(f"  queued rep{i:02d}  seed={seed}  job_id={resp['job_id']}")
+
+
+if __name__ == "__main__":
+    main()

v3/scripts/main/phase5/dispatch_phase5.py

@@ -0,0 +1,199 @@
+"""
+Dispatch phase 5 experiment runs to the distributed job server.
+
+Reads experiment_grid.json, checks which runs already have complete 10x5 results,
+and submits the rest. Skips runs marked needs_implementation.
+
+Usage:
+    python -m v3.scripts.main.phase5.dispatch_phase5 \
+        --server http://hades:8765 --token hypertower
+
+    # Dry run (print what would be submitted, don't actually submit):
+    python -m v3.scripts.main.phase5.dispatch_phase5 \
+        --server http://hades:8765 --token hypertower --dry-run
+
+    # Override number of reps (default 10):
+    python -m v3.scripts.main.phase5.dispatch_phase5 \
+        --server http://hades:8765 --token hypertower --reps 4
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+import requests
+
+# Allow running as `python v3/scripts/main/phase3/dispatch_phase3.py`
+sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
+
+GRID_PATH   = Path(__file__).parent / "experiment_grid.json"
+RESULTS_ROOT = Path(__file__).resolve().parents[4] / "v3" / "results"
+MODULE      = "v3.scripts.main.run_cv"
+OUTPUT_DIR  = "v3/results"
+REP_SEED_START = 100
+REP_SEED_STEP  = 100
+
+
+# ── Completion check ──────────────────────────────────────────────────────────
+
+def _completed_reps(run_name: str, reps: int) -> list[int]:
+    """Return list of rep indices that already have a summary.json (any tower mode)."""
+    done = []
+    for i in range(reps):
+        rep_dir = RESULTS_ROOT / run_name / f"rep{i:02d}" / "binary"
+        # Accept any tower mode subdir
+        if rep_dir.exists() and any((rep_dir / tm / "summary.json").exists()
+                                     for tm in ("single", "bilateral", "siamese", "ensemble")):
+            done.append(i)
+    return done
+
+
+# ── Server API ────────────────────────────────────────────────────────────────
+
+class _API:
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self._h = {"x-token": token}
+
+    def get(self, path: str, **params) -> object:
+        r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+    def post(self, path: str, body: dict) -> dict:
+        r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+
+def _queued_reps(jobs: list[dict], run_name: str) -> set[int]:
+    """Return rep indices already pending or running in the server queue."""
+    active = set()
+    for job in jobs:
+        if job["run_name"] != run_name:
+            continue
+        if job["state"] not in ("pending", "running"):
+            continue
+        # Extract --rep-index from job args
+        try:
+            args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"])
+            if "--rep-index" in args:
+                active.add(int(args[args.index("--rep-index") + 1]))
+        except Exception:
+            pass
+    return active
+
+
+def _submit_cv(api: _API, run_name: str, run_args: list[str],
+               reps: int, missing: list[int], dry_run: bool):
+    """Submit one job per missing rep."""
+    for i in missing:
+        seed = REP_SEED_START + i * REP_SEED_STEP
+        rep_args = run_args + [
+            "--run-name", run_name,
+            "--reps", "1",
+            "--rep-seed-start", str(seed),
+            "--rep-index", str(i),
+        ]
+        body = {
+            "run_name": run_name,
+            "module": MODULE,
+            "args": rep_args,
+            "output_dir": OUTPUT_DIR,
+            "priority": 0,
+        }
+        if dry_run:
+            print(f"  [dry-run] would queue rep{i:02d}  seed={seed}")
+        else:
+            resp = api.post("/jobs", body)
+            print(f"  queued rep{i:02d}  seed={seed}  job_id={resp['job_id']}")
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                  formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""),
+                    help="Server URL (or set HT_SERVER)")
+    ap.add_argument("--token",  default=os.environ.get("HT_TOKEN",  ""),
+                    help="Shared secret (or set HT_TOKEN)")
+    ap.add_argument("--reps",   type=int, default=10,
+                    help="Expected number of reps per run (default: 10)")
+    ap.add_argument("--grid",   type=Path, default=GRID_PATH,
+                    help="Path to experiment grid JSON (default: experiment_grid.json)")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="Print what would be submitted without actually submitting")
+    args = ap.parse_args()
+
+    if not args.dry_run:
+        if not args.server:
+            ap.error("--server is required (or set HT_SERVER)")
+        if not args.token:
+            ap.error("--token is required (or set HT_TOKEN)")
+    elif not args.server or not args.token:
+        print("[dry-run] no --server/--token provided — skipping queue check, showing disk state only")
+
+    grid = json.loads(args.grid.read_text())
+    common_args = grid["common_args"]
+    api = _API(args.server, args.token) if (args.server and args.token) else None
+
+    # Fetch current server queue once (pending + running)
+    server_jobs: list[dict] = []
+    if api:
+        try:
+            all_jobs = api.get("/jobs")
+            server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")]
+            print(f"[server] {len(server_jobs)} job(s) currently pending/running in queue")
+        except Exception as e:
+            print(f"[warn] could not fetch server queue: {e}")
+
+    # Collect all runs: baseline + every group's runs
+    all_runs = [grid["baseline"]]
+    for group in grid["groups"]:
+        if group.get("needs_implementation"):
+            print(f"\n[skip] group '{group['name']}' — {group['needs_implementation']}")
+            continue
+        all_runs.extend(group["runs"])
+
+    submitted_total = 0
+    skipped_total   = 0
+
+    for run in all_runs:
+        run_name  = run["run_name"]
+        run_args  = common_args + run.get("extra_args", [])
+        done      = set(_completed_reps(run_name, args.reps))
+        queued    = _queued_reps(server_jobs, run_name)
+        accounted = done | queued
+        missing   = [i for i in range(args.reps) if i not in accounted]
+
+        if not missing:
+            if len(done) == args.reps:
+                print(f"\n[done]  {run_name}  ({args.reps}/{args.reps} reps complete)")
+            else:
+                in_q = sorted(queued - done)
+                print(f"\n[skip]  {run_name}  ({len(done)} done, {len(in_q)} queued: {[f'rep{i:02d}' for i in in_q]})")
+            skipped_total += 1
+            continue
+
+        parts = []
+        if done:    parts.append(f"{len(done)} done")
+        if queued:  parts.append(f"{len(queued - done)} queued")
+        status = ", ".join(parts) if parts else "not started"
+        print(f"\n[queue] {run_name}  ({status}) — submitting {len(missing)} rep(s)")
+        _submit_cv(api, run_name, run_args, args.reps, missing, args.dry_run)
+        submitted_total += len(missing)
+
+    print(f"\n{'='*50}")
+    print(f"Submitted: {submitted_total} jobs  |  Already accounted for: {skipped_total} runs")
+    if grid.get("groups"):
+        needs_impl = sum(1 for g in grid["groups"] if g.get("needs_implementation"))
+        if needs_impl:
+            print(f"Skipped (needs implementation): {needs_impl} group(s)")
+
+
+if __name__ == "__main__":
+    main()

v3/scripts/main/phase5/experiment_grid.json

@@ -0,0 +1,87 @@
+{
+  "_notes": [
+    "Phase 5 — Full HyperTower: bilateral + clinical data + aggregation strategy comparison.",
+    "Goal: show the effect of a dual CNN + clinical data, and compare ensemble vs fused-head.",
+    "Best settings from all prior phases: refugelike, iop_ratio_drop_raw, bcd_p05.",
+    "Best bilateral architecture from phase 4 should be used — update tower-mode accordingly.",
+    "common_args are prepended to every run's args list.",
+    "NOTE: update --tower-mode in groups below once phase 4 winner is known.",
+    "Placeholder uses 'bilateral' — change to 'siamese' if that wins phase 4."
+  ],
+
+  "common_args": [
+    "--eval-mode", "binary",
+    "--bridge-mode", "fused",
+    "--epochs", "30",
+    "--in-memory-cache",
+    "--augment",
+    "--tune-binary-threshold",
+    "--backbone", "refugelike",
+    "--iop-corr-method", "ratio",
+    "--iop-drop-raw",
+    "--exclude-cols", "Axial_Length",
+    "--output-root", "v3/results"
+  ],
+
+  "_common_args_implicit_defaults": {
+    "--tower-loss-mode": "bcd",
+    "--bcd-prob": "0.5",
+    "--warmup-cd-epochs": "40",
+    "--single-warmup-tower-epochs": "3",
+    "--single-warmup-fused-epochs": "3",
+    "--bilat-warmup-tower-epochs": "3",
+    "--bilat-warmup-fused-epochs": "3"
+  },
+
+  "baseline": {
+    "run_name": "phase5/single_fused",
+    "description": "Single-eye + clinical data — phase 3 best config, re-run as direct comparison baseline for phase 5.",
+    "extra_args": ["--tower-mode", "single"]
+  },
+
+  "groups": [
+    {
+      "name": "bilateral_clinical",
+      "description": "Add clinical data to bilateral architectures.",
+      "runs": [
+        {
+          "run_name": "phase5/ensemble_fused",
+          "description": "Ensemble (independent OD+OS) + clinical data via fused bridge.",
+          "extra_args": ["--tower-mode", "ensemble"]
+        },
+        {
+          "run_name": "phase5/bilateral_fused",
+          "description": "BilateralHT + clinical data — full canonical HyperTower.",
+          "extra_args": ["--tower-mode", "bilateral"]
+        },
+        {
+          "run_name": "phase5/siamese_fused",
+          "description": "SiameseHT + clinical data — siamese mean+delta with fused clinical bridge.",
+          "extra_args": ["--tower-mode", "siamese"]
+        }
+      ]
+    },
+
+    {
+      "name": "aggregation",
+      "description": "Compare patient-level prediction aggregation strategies on top of ensemble.",
+      "runs": [
+        {
+          "run_name": "phase5/ensemble_fused_head",
+          "description": "Ensemble + clinical data + attention scorer head (Linear(C→1) per eye, softmax-weighted average).",
+          "extra_args": ["--tower-mode", "ensemble", "--fused-head"]
+        },
+        {
+          "run_name": "phase5/logit_mlp_head",
+          "description": "Ensemble + clinical data + logit-level MLP head (cat([logit_od, logit_os]) → FC(64) → FC(C)).",
+          "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--head-type", "logit_mlp"]
+        },
+        {
+          "run_name": "phase5/embedding_mlp_head",
+          "description": "Ensemble + clinical data + embedding-level MLP head (cat([z_od, z_os]) → FC(256) → FC(C)).",
+          "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--head-type", "embedding_mlp"]
+        }
+      ]
+    }
+  ]
+}

v3/scripts/main/phase6/dispatch_phase6a.py

@@ -0,0 +1,204 @@
+"""
+Dispatch phase 6a experiment runs (geometry vector injection) to the distributed job server.
+
+Covers the geometry_vector_gt and geometry_vector_unet groups from experiment_grid.json:
+  - GT geometry vector × {single, ensemble, fused-head}
+  - U-Net geometry vector × {single, ensemble, fused-head}
+
+Skips geometry_tower_* groups (needs_implementation — will get dispatch_phase6b.py).
+
+Usage:
+    python -m v3.scripts.main.phase6.dispatch_phase6a \
+        --server http://hades:8765 --token hypertower
+
+    # Dry run (print what would be submitted, don't actually submit):
+    python -m v3.scripts.main.phase6.dispatch_phase6a \
+        --server http://hades:8765 --token hypertower --dry-run
+
+    # Override number of reps (default 10):
+    python -m v3.scripts.main.phase6.dispatch_phase6a \
+        --server http://hades:8765 --token hypertower --reps 4
+
+NOTE: Requires --geometry-dim and --geometry-source to be wired into
+v3_hypertower.py before these jobs will run successfully.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+import requests
+
+# Allow running as `python v3/scripts/main/phase6/dispatch_phase6a.py`
+sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
+
+GRID_PATH    = Path(__file__).parent / "experiment_grid.json"
+RESULTS_ROOT = Path(__file__).resolve().parents[4] / "v3" / "results"
+MODULE       = "v3.scripts.main.run_cv"
+OUTPUT_DIR   = "v3/results"
+REP_SEED_START = 100
+REP_SEED_STEP  = 100
+
+
+# ── Completion check ──────────────────────────────────────────────────────────
+
+def _completed_reps(run_name: str, reps: int) -> list[int]:
+    """Return list of rep indices that already have a summary.json (any tower mode)."""
+    done = []
+    for i in range(reps):
+        rep_dir = RESULTS_ROOT / run_name / f"rep{i:02d}" / "binary"
+        # Accept any tower mode subdir
+        if rep_dir.exists() and any((rep_dir / tm / "summary.json").exists()
+                                     for tm in ("single", "bilateral", "siamese", "ensemble", "tri", "tri_bilateral")):
+            done.append(i)
+    return done
+
+
+# ── Server API ────────────────────────────────────────────────────────────────
+
+class _API:
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self._h = {"x-token": token}
+
+    def get(self, path: str, **params) -> object:
+        r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+    def post(self, path: str, body: dict) -> dict:
+        r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+
+def _queued_reps(jobs: list[dict], run_name: str) -> set[int]:
+    """Return rep indices already pending or running in the server queue."""
+    active = set()
+    for job in jobs:
+        if job["run_name"] != run_name:
+            continue
+        if job["state"] not in ("pending", "running"):
+            continue
+        try:
+            args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"])
+            if "--rep-index" in args:
+                active.add(int(args[args.index("--rep-index") + 1]))
+        except Exception:
+            pass
+    return active
+
+
+def _submit_cv(api: _API, run_name: str, run_args: list[str],
+               reps: int, missing: list[int], dry_run: bool):
+    """Submit one job per missing rep."""
+    for i in missing:
+        seed = REP_SEED_START + i * REP_SEED_STEP
+        rep_args = run_args + [
+            "--run-name", run_name,
+            "--reps", "1",
+            "--rep-seed-start", str(seed),
+            "--rep-index", str(i),
+        ]
+        body = {
+            "run_name": run_name,
+            "module": MODULE,
+            "args": rep_args,
+            "output_dir": OUTPUT_DIR,
+            "priority": 0,
+        }
+        if dry_run:
+            print(f"  [dry-run] would queue rep{i:02d}  seed={seed}")
+        else:
+            resp = api.post("/jobs", body)
+            print(f"  queued rep{i:02d}  seed={seed}  job_id={resp['job_id']}")
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                  formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""),
+                    help="Server URL (or set HT_SERVER)")
+    ap.add_argument("--token",  default=os.environ.get("HT_TOKEN",  ""),
+                    help="Shared secret (or set HT_TOKEN)")
+    ap.add_argument("--reps",   type=int, default=10,
+                    help="Expected number of reps per run (default: 10)")
+    ap.add_argument("--grid",   type=Path, default=GRID_PATH,
+                    help="Path to experiment grid JSON (default: experiment_grid.json)")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="Print what would be submitted without actually submitting")
+    args = ap.parse_args()
+
+    if not args.dry_run:
+        if not args.server:
+            ap.error("--server is required (or set HT_SERVER)")
+        if not args.token:
+            ap.error("--token is required (or set HT_TOKEN)")
+    elif not args.server or not args.token:
+        print("[dry-run] no --server/--token provided — skipping queue check, showing disk state only")
+
+    grid = json.loads(args.grid.read_text())
+    common_args = grid["common_args"]
+    api = _API(args.server, args.token) if (args.server and args.token) else None
+
+    # Fetch current server queue once (pending + running)
+    server_jobs: list[dict] = []
+    if api:
+        try:
+            all_jobs = api.get("/jobs")
+            server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")]
+            print(f"[server] {len(server_jobs)} job(s) currently pending/running in queue")
+        except Exception as e:
+            print(f"[warn] could not fetch server queue: {e}")
+
+    # Collect all runs: baseline + every group's runs (skip needs_implementation)
+    all_runs = [grid["baseline"]]
+    for group in grid["groups"]:
+        if group.get("needs_implementation"):
+            print(f"\n[skip] group '{group['name']}' — {group['needs_implementation']}")
+            continue
+        all_runs.extend(group["runs"])
+
+    submitted_total = 0
+    skipped_total   = 0
+
+    for run in all_runs:
+        run_name  = run["run_name"]
+        run_args  = common_args + run.get("extra_args", [])
+        done      = set(_completed_reps(run_name, args.reps))
+        queued    = _queued_reps(server_jobs, run_name)
+        accounted = done | queued
+        missing   = [i for i in range(args.reps) if i not in accounted]
+
+        if not missing:
+            if len(done) == args.reps:
+                print(f"\n[done]  {run_name}  ({args.reps}/{args.reps} reps complete)")
+            else:
+                in_q = sorted(queued - done)
+                print(f"\n[skip]  {run_name}  ({len(done)} done, {len(in_q)} queued: {[f'rep{i:02d}' for i in in_q]})")
+            skipped_total += 1
+            continue
+
+        parts = []
+        if done:    parts.append(f"{len(done)} done")
+        if queued:  parts.append(f"{len(queued - done)} queued")
+        status = ", ".join(parts) if parts else "not started"
+        print(f"\n[queue] {run_name}  ({status}) — submitting {len(missing)} rep(s)")
+        _submit_cv(api, run_name, run_args, args.reps, missing, args.dry_run)
+        submitted_total += len(missing)
+
+    print(f"\n{'='*50}")
+    print(f"Submitted: {submitted_total} jobs  |  Already accounted for: {skipped_total} runs")
+    if grid.get("groups"):
+        needs_impl = sum(1 for g in grid["groups"] if g.get("needs_implementation"))
+        if needs_impl:
+            print(f"Skipped (needs implementation): {needs_impl} group(s)")
+
+
+if __name__ == "__main__":
+    main()

v3/scripts/main/phase6/experiment_grid.json

@@ -0,0 +1,144 @@
+{
+  "_notes": [
+    "Phase 6 — Geometry augmentation: vector injection and dedicated geometry tower.",
+    "Goal: test whether derived structural geometry (CDR, rim ratio, etc.) improves performance",
+    "  injected either as a 5-dim vector appended to the clinical stream (Part A)",
+    "  or as a dedicated geometry tower feeding into bridge fusion (Part B).",
+    "Tower modes under test: single, ensemble, fused-head (ensemble + --fused-head).",
+    "Geometry sources: GT contour annotations (no annotator-bias concern at this stage —",
+    "  labels were assigned by same clinicians, GT seg merely measures CDR directly)",
+    "  and U-Net segmentations (REFUGE-trained, fine-tuned on PAPILA folds — unbiased).",
+    "Part A (geometry_vector): vector injection — dispatch_phase6a.py covers this.",
+    "  Requires: --geometry-dim and --geometry-source wired into v3_hypertower.py.",
+    "Part B (geometry_tower): dedicated geometry tower — needs architecture implementation.",
+    "  Will get its own dispatch_phase6b.py once built.",
+    "Ensemble and fused-head baselines come from phase5. Single has no equivalent with all tuned",
+    "  hyperparameters, so a phase6 single baseline is included here.",
+    "Best settings from all prior phases: refugelike backbone, iop_ratio_drop_raw, bcd_p05.",
+    "Geometry features: [area_cdr, rim_ratio, vertical_cdr, horizontal_cdr, centre_shift] (dim=5).",
+    "common_args are prepended to every run's args list."
+  ],
+
+  "common_args": [
+    "--eval-mode", "binary",
+    "--bridge-mode", "fused",
+    "--epochs", "30",
+    "--in-memory-cache",
+    "--augment",
+    "--tune-binary-threshold",
+    "--backbone", "refugelike",
+    "--iop-corr-method", "ratio",
+    "--iop-drop-raw",
+    "--exclude-cols", "Axial_Length",
+    "--img-crop-manifest", "manifest.csv",
+    "--output-root", "v3/results"
+  ],
+
+  "_common_args_implicit_defaults": {
+    "--tower-loss-mode": "bcd",
+    "--bcd-prob": "0.5",
+    "--warmup-cd-epochs": "40",
+    "--single-warmup-tower-epochs": "3",
+    "--single-warmup-fused-epochs": "3",
+    "--bilat-warmup-tower-epochs": "3",
+    "--bilat-warmup-fused-epochs": "3"
+  },
+
+  "baseline": {
+    "run_name": "phase6/single_no_geom",
+    "description": "Single-eye + clinical data, no geometry — needed as Phase 6 baseline since no prior phase ran single with all tuned hyperparameters (iop_ratio_drop_raw, bcd_p05, refugelike).",
+    "extra_args": ["--tower-mode", "single"]
+  },
+
+  "groups": [
+    {
+      "name": "geometry_vector_gt",
+      "description": "Part A — Inject 5-dim geometry vector from GT annotations alongside clinical data. Three aggregation modes: single-eye, ensemble (independent OD+OS), ensemble with fused head.",
+      "runs": [
+        {
+          "run_name": "phase6/vec_gt_single",
+          "description": "Single-eye + clinical + GT geometry vector appended to clinical stream.",
+          "extra_args": ["--tower-mode", "single", "--geometry-dim", "5", "--geometry-source", "gt"]
+        },
+        {
+          "run_name": "phase6/vec_gt_ensemble",
+          "description": "Ensemble + clinical + GT geometry vector (per-eye geometry, independent OD+OS mean).",
+          "extra_args": ["--tower-mode", "ensemble", "--geometry-dim", "5", "--geometry-source", "gt"]
+        },
+        {
+          "run_name": "phase6/vec_gt_fused_head",
+          "description": "Ensemble + fused head + clinical + GT geometry vector.",
+          "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--geometry-dim", "5", "--geometry-source", "gt"]
+        }
+      ]
+    },
+
+    {
+      "name": "geometry_vector_unet",
+      "description": "Part A — Same three modes but geometry from U-Net segmentations (REFUGE-trained, fine-tuned). Tests whether GT annotator bias affects the geometry signal.",
+      "runs": [
+        {
+          "run_name": "phase6/vec_unet_single",
+          "description": "Single-eye + clinical + U-Net geometry vector.",
+          "extra_args": ["--tower-mode", "single", "--geometry-dim", "5", "--geometry-source", "unet"]
+        },
+        {
+          "run_name": "phase6/vec_unet_ensemble",
+          "description": "Ensemble + clinical + U-Net geometry vector.",
+          "extra_args": ["--tower-mode", "ensemble", "--geometry-dim", "5", "--geometry-source", "unet"]
+        },
+        {
+          "run_name": "phase6/vec_unet_fused_head",
+          "description": "Ensemble + fused head + clinical + U-Net geometry vector.",
+          "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--geometry-dim", "5", "--geometry-source", "unet"]
+        }
+      ]
+    },
+
+    {
+      "name": "geometry_tower_gt",
+      "description": "Part B — Dedicated geometry tower (MLP on 5-dim vector, separate from clinical tower) with GT geometry. Requires tri-tower bridge architecture.",
+      "needs_implementation": "Geometry tower not yet built — requires GeometryTower MLP, bridge reconfiguration for n>=3 towers, and --geometry-tower CLI flag.",
+      "runs": [
+        {
+          "run_name": "phase6/tower_gt_single",
+          "description": "Single-eye + clinical tower + dedicated GT geometry tower.",
+          "extra_args": ["--tower-mode", "single", "--geometry-tower", "--geometry-source", "gt"]
+        },
+        {
+          "run_name": "phase6/tower_gt_ensemble",
+          "description": "Ensemble + clinical tower + dedicated GT geometry tower.",
+          "extra_args": ["--tower-mode", "ensemble", "--geometry-tower", "--geometry-source", "gt"]
+        },
+        {
+          "run_name": "phase6/tower_gt_fused_head",
+          "description": "Ensemble + fused head + clinical tower + dedicated GT geometry tower.",
+          "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--geometry-tower", "--geometry-source", "gt"]
+        }
+      ]
+    },
+
+    {
+      "name": "geometry_tower_unet",
+      "description": "Part B — Same three modes with dedicated geometry tower, U-Net geometry source.",
+      "needs_implementation": "Geometry tower not yet built — requires GeometryTower MLP, bridge reconfiguration for n>=3 towers, and --geometry-tower CLI flag.",
+      "runs": [
+        {
+          "run_name": "phase6/tower_unet_single",
+          "description": "Single-eye + clinical tower + dedicated U-Net geometry tower.",
+          "extra_args": ["--tower-mode", "single", "--geometry-tower", "--geometry-source", "unet"]
+        },
+        {
+          "run_name": "phase6/tower_unet_ensemble",
+          "description": "Ensemble + clinical tower + dedicated U-Net geometry tower.",
+          "extra_args": ["--tower-mode", "ensemble", "--geometry-tower", "--geometry-source", "unet"]
+        },
+        {
+          "run_name": "phase6/tower_unet_fused_head",
+          "description": "Ensemble + fused head + clinical tower + dedicated U-Net geometry tower.",
+          "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--geometry-tower", "--geometry-source", "unet"]
+        }
+      ]
+    }
+  ]
+}

v3/scripts/main/run_cv.py

@@ -54,6 +54,11 @@ def build_parser() -> argparse.ArgumentParser:
         "--rep-seed-step", type=int, default=100,
         help="Increment between rep fold-seeds (default: 100; rep k uses seed start + k*step).",
     )
+    ap.add_argument(
+        "--rep-index", type=int, default=None,
+        help="Override the rep directory index (e.g. 3 → rep03). "
+             "Used by the distributed server to run a single rep of a multi-rep job.",
+    )
     return ap
 
 
@@ -65,22 +70,24 @@ def main():
     seed_start     = int(args.rep_seed_start)
     seed_step      = int(args.rep_seed_step)
     base_run_name  = args.run_name or "v3_cv"
+    rep_index_override = getattr(args, "rep_index", None)
 
     for rep in range(reps):
         rep_seed = seed_start + rep * seed_step
         args.fold_seed = rep_seed
 
-        if reps > 1:
+        dir_index = rep_index_override if (rep_index_override is not None and reps == 1) else rep
-            args.run_name = f"{base_run_name}/rep{rep:02d}"
+        if reps > 1 or rep_index_override is not None:
+            args.run_name = f"{base_run_name}/rep{dir_index:02d}"
             print(f"\n{'='*60}", flush=True)
-            print(f"Rep {rep+1}/{reps}  fold_seed={rep_seed}", flush=True)
+            print(f"Rep {dir_index+1}  fold_seed={rep_seed}", flush=True)
             print(f"{'='*60}", flush=True)
         else:
             args.run_name = base_run_name
 
         tower = V3HyperTower(args)
         out_dir = tower.run()
-        print(f"\nRep {rep+1} output: {out_dir}", flush=True)
+        print(f"\nRep {dir_index+1} output: {out_dir}", flush=True)
 
 
 if __name__ == "__main__":

v3/scripts/output_analysis/analyze_phases.py

@@ -0,0 +1,809 @@
+"""
+Phase 1–5 analysis — ablation plots + pairwise Wilcoxon tests.
+
+Usage:
+    python -m v3.scripts.output_analysis.analyze_phases          # all phases
+    python -m v3.scripts.output_analysis.analyze_phases --phase 3
+    python -m v3.scripts.output_analysis.analyze_phases --out figures/
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from scipy.stats import wilcoxon
+from sklearn.metrics import roc_auc_score
+from statsmodels.stats.multitest import multipletests
+
+RESULTS_ROOT = Path(__file__).resolve().parents[3] / "v3" / "results"
+
+# Shared style constants
+C_BASELINE = "#dd8452"
+C_OTHER    = "#4c72b0"
+C_MEDIAN   = "#c44e52"
+FSIZE      = 10
+
+
+# ── Helpers ──────────────────────────────────────────────────────────────────
+
+def load_rep_aucs(run_path: Path, test_key: str = "classic_test") -> np.ndarray:
+    """Return array of per-rep AUC means for a run directory."""
+    aucs = []
+    for rep_dir in sorted(run_path.glob("rep*")):
+        for summary in rep_dir.rglob("summary.json"):
+            txt = summary.read_text().strip()
+            if not txt:
+                continue
+            d = json.loads(txt)
+            auc = d.get("mode_summary", {}).get(test_key, {}).get("auc_mean")
+            if auc is not None:
+                aucs.append(auc)
+                break  # one summary per rep
+    return np.array(aucs)
+
+
+def wilcoxon_p(a: np.ndarray, b: np.ndarray) -> float:
+    """Two-sided Wilcoxon signed-rank p-value; returns nan if underpowered."""
+    diffs = a - b
+    if np.all(diffs == 0) or len(diffs) < 5:
+        return float("nan")
+    try:
+        return wilcoxon(diffs, alternative="two-sided").pvalue
+    except Exception:
+        return float("nan")
+
+
+def stars(p: float) -> str:
+    if np.isnan(p):   return ""
+    if p < 0.001:     return "***"
+    if p < 0.01:      return "**"
+    if p < 0.05:      return "*"
+    return "ns"
+
+
+def paired_matrix(runs: list[str], aucs_dict: dict[str, np.ndarray],
+                  fdr: bool = True) -> tuple[np.ndarray, np.ndarray]:
+    """Return (p_matrix, corrected_p_matrix) shape (n, n)."""
+    n = len(runs)
+    raw = np.full((n, n), np.nan)
+    for i, a in enumerate(runs):
+        for j, b in enumerate(runs):
+            if i != j and a in aucs_dict and b in aucs_dict:
+                ai, bi = aucs_dict[a], aucs_dict[b]
+                min_n = min(len(ai), len(bi))
+                if min_n >= 5:
+                    raw[i, j] = wilcoxon_p(ai[:min_n], bi[:min_n])
+    if fdr:
+        mask = ~np.isnan(raw)
+        if mask.sum() > 0:
+            flat = raw[mask]
+            _, corrected, _, _ = multipletests(flat, method="fdr_bh")
+            corr = raw.copy()
+            corr[mask] = corrected
+            return raw, corr
+    return raw, raw.copy()
+
+
+def boxplot_panel(ax, data_list, labels, base_idx, title="", ylabel="Test AUC",
+                  base_aucs=None, all_aucs_by_label=None):
+    """Vertical box plot with p-value vs baseline under each tick label."""
+    n = len(labels)
+    x = np.arange(n)
+    colors = [C_BASELINE if i == base_idx else C_OTHER for i in range(n)]
+
+    bp = ax.boxplot(data_list, vert=True, patch_artist=True, positions=x,
+                    widths=0.3, showfliers=True,
+                    flierprops=dict(marker="o", markersize=3, alpha=0.5),
+                    medianprops=dict(color=C_MEDIAN, linewidth=2))
+    for patch, color in zip(bp["boxes"], colors):
+        patch.set_facecolor(color)
+        patch.set_alpha(0.8)
+
+    if base_aucs is not None:
+        ax.axhline(np.median(base_aucs), color=C_BASELINE, linewidth=1,
+                   linestyle="--", alpha=0.5, label="Baseline median")
+        ax.legend(fontsize=FSIZE - 1)
+
+    ax.set_xlim(-0.5, n - 0.5)
+    ax.set_ylabel(ylabel, fontsize=FSIZE + 1)
+    if title:
+        ax.set_title(title, fontsize=FSIZE + 1, fontweight="bold")
+    ax.grid(axis="y", alpha=0.3)
+
+    tick_labels = []
+    for i, lbl in enumerate(labels):
+        if i == base_idx or base_aucs is None:
+            tick_labels.append(lbl)
+            continue
+        a = (all_aucs_by_label or {}).get(lbl, data_list[i])
+        min_n = min(len(a), len(base_aucs))
+        p = wilcoxon_p(a[:min_n], base_aucs[:min_n])
+        p_str = f"p={p:.3f}" if not np.isnan(p) else "p=n/a"
+        tick_labels.append(f"{lbl}\n{p_str}")
+
+    ax.set_xticks(x)
+    ax.set_xticklabels(tick_labels, fontsize=FSIZE)
+
+
+def bar_plot(ax, labels, means, stds, baseline_idx, title, ylabel="AUC",
+             baseline_aucs=None, all_aucs=None):
+    """Horizontal bar chart with baseline highlighted and p-value annotations."""
+    n = len(labels)
+    colors = ["#4c72b0" if i != baseline_idx else "#dd8452" for i in range(n)]
+    y = np.arange(n)
+    bars = ax.barh(y, means, xerr=stds, color=colors, alpha=0.85,
+                   height=0.6, capsize=3, error_kw=dict(linewidth=1))
+    ax.set_yticks(y)
+    ax.set_yticklabels(labels, fontsize=8)
+    ax.set_xlabel(ylabel)
+    ax.set_title(title, fontsize=10, fontweight="bold")
+    ax.axvline(means[baseline_idx], color="#dd8452", linewidth=1, linestyle="--", alpha=0.6)
+
+    # Annotate with p-value stars vs baseline
+    if baseline_aucs is not None and all_aucs is not None:
+        x_max = max(means) + max(stds) + 0.005
+        for i, lbl in enumerate(labels):
+            if i == baseline_idx:
+                continue
+            a = all_aucs.get(lbl)
+            if a is None:
+                continue
+            min_n = min(len(a), len(baseline_aucs))
+            p = wilcoxon_p(a[:min_n], baseline_aucs[:min_n])
+            s = stars(p)
+            if s:
+                ax.text(x_max, i, s, va="center", fontsize=7,
+                        color="black" if s != "ns" else "gray")
+
+
+def pairwise_heatmap(ax, runs, p_matrix, title):
+    """Lower-triangle heatmap of corrected p-values."""
+    n = len(runs)
+    display = np.full_like(p_matrix, np.nan)
+    for i in range(n):
+        for j in range(i):
+            display[i, j] = p_matrix[i, j]
+
+    im = ax.imshow(display, vmin=0, vmax=0.1, cmap="RdYlGn_r", aspect="auto")
+    ax.set_xticks(range(n))
+    ax.set_yticks(range(n))
+    ax.set_xticklabels(runs, rotation=45, ha="right", fontsize=7)
+    ax.set_yticklabels(runs, fontsize=7)
+    ax.set_title(title, fontsize=10, fontweight="bold")
+    plt.colorbar(im, ax=ax, label="p-value (FDR)")
+
+    for i in range(n):
+        for j in range(i):
+            p = display[i, j]
+            if not np.isnan(p):
+                ax.text(j, i, f"{p:.2f}", ha="center", va="center",
+                        fontsize=6, color="white" if p < 0.05 else "black")
+
+
+# ── Phase 1 ──────────────────────────────────────────────────────────────────
+
+P1_CLF_ORDER  = ["KNN", "Random Forest", "SVM", "Logistic Regression"]
+P1_CLF_LABELS = {"KNN": "KNN", "Random Forest": "RF", "SVM": "SVM", "Logistic Regression": "LR"}
+P1_TAGS = [
+    ("no_leakage",        "baseline"),
+    ("hypertower_loader", "HT loader"),
+]
+P1_BASELINE_TAG = "no_leakage"
+P1_PAPER_AUC = {"KNN": 0.75, "Random Forest": 0.64, "SVM": 0.75, "Logistic Regression": 0.70}
+
+P1_BACKBONES = ["densenet121", "vgg16", "mobilenet_v2", "inception_v3", "resnet50"]
+P1_BACKBONE_LABELS = {
+    "densenet121":  "DenseNet121",
+    "vgg16":        "VGG16",
+    "mobilenet_v2": "MobileNetV2",
+    "inception_v3": "InceptionV3",
+    "resnet50":     "ResNet50",
+}
+P1_PAPER_CNN = {
+    "densenet121": (0.80, 0.05), "vgg16": (0.84, 0.02),
+    "mobilenet_v2": (0.75, 0.06), "inception_v3": (0.78, 0.08), "resnet50": (0.78, 0.07),
+}
+
+
+def _load_p1_clf_aucs(phase1_dir: Path) -> dict:
+    """Load per-fold AUCs for each tag × classifier."""
+    data = {}
+    for tag, _ in P1_TAGS:
+        data[tag] = {}
+        for clf in P1_CLF_ORDER:
+            fpath = phase1_dir / tag / clf / "fold_metrics.csv"
+            if fpath.exists():
+                data[tag][clf] = pd.read_csv(fpath)["auc"].tolist()
+    return data
+
+
+def _load_p1_cnn_aucs(phase1_dir: Path) -> tuple[dict, dict]:
+    cnn, ht = {}, {}
+    for b in P1_BACKBONES:
+        fpath = phase1_dir / f"cnn_{b}" / "fold_metrics.csv"
+        cnn[b] = pd.read_csv(fpath)["auc"].tolist() if fpath.exists() else []
+        aucs = []
+        for fold in range(5):
+            yp = phase1_dir / "imageonly_ht" / b / "binary" / "single" / f"fold{fold}" / "test_y_true.npy"
+            pp = phase1_dir / "imageonly_ht" / b / "binary" / "single" / f"fold{fold}" / "test_probs_fused.npy"
+            if yp.exists() and pp.exists():
+                y, pr = np.load(yp), np.load(pp)
+                if len(np.unique(y)) >= 2:
+                    aucs.append(float(roc_auc_score(y, pr[:, 1])))
+        ht[b] = aucs
+    return cnn, ht
+
+
+def analyze_phase1(out_dir: Path):
+    print("\n=== Phase 1 ===")
+    phase1_dir = RESULTS_ROOT / "phase1"
+
+    # ── Clinical classifiers ────────────────────────────────────────────────
+    clf_data = _load_p1_clf_aucs(phase1_dir)
+    n_clf  = len(P1_CLF_ORDER)
+    n_tags = len(P1_TAGS)
+    group_w = 0.7
+    box_w   = group_w / n_tags * 0.85
+    offsets = np.linspace(-group_w / 2 + box_w / 2, group_w / 2 - box_w / 2, n_tags)
+    tag_colors = [C_BASELINE if t == P1_BASELINE_TAG else C_OTHER for t, _ in P1_TAGS]
+
+    fig1, ax1 = plt.subplots(figsize=(10, 5))
+    fig1.suptitle("Phase 1 — Clinical-only classifiers: CV strategy comparison",
+                  fontsize=FSIZE + 2, fontweight="bold")
+
+    for ti, (tag, lbl) in enumerate(P1_TAGS):
+        color = tag_colors[ti]
+        first = True
+        for ci, clf in enumerate(P1_CLF_ORDER):
+            aucs = clf_data.get(tag, {}).get(clf, [])
+            if not aucs:
+                continue
+            bp = ax1.boxplot(aucs, positions=[ci + offsets[ti]], widths=box_w,
+                             patch_artist=True, manage_ticks=False,
+                             boxprops=dict(facecolor=color, alpha=0.8),
+                             medianprops=dict(color=C_MEDIAN, linewidth=2),
+                             whiskerprops=dict(color=color, linewidth=1.2),
+                             capprops=dict(color=color, linewidth=1.2),
+                             flierprops=dict(marker="o", markersize=3, alpha=0.5))
+            if first:
+                bp["boxes"][0].set_label(lbl)
+                first = False
+
+    for ci, clf in enumerate(P1_CLF_ORDER):
+        if clf in P1_PAPER_AUC:
+            ax1.hlines(P1_PAPER_AUC[clf], ci - group_w / 2, ci + group_w / 2,
+                       colors="black", linestyles=":", linewidths=1.5,
+                       label="PAPILA paper" if ci == 0 else "_nolegend_")
+
+    ax1.set_xticks(range(n_clf))
+    ax1.set_xticklabels([P1_CLF_LABELS[c] for c in P1_CLF_ORDER], fontsize=FSIZE + 1)
+    ax1.set_ylabel("Test AUC", fontsize=FSIZE + 1)
+    ax1.set_ylim(0.45, 1.02)
+    ax1.axhline(0.5, color="grey", linestyle="--", linewidth=0.8, alpha=0.4)
+    ax1.grid(axis="y", alpha=0.3)
+    ax1.legend(fontsize=FSIZE, loc="lower right", framealpha=0.9)
+    fig1.tight_layout()
+    p1 = out_dir / "phase1_clinical_classifiers.png"
+    fig1.savefig(p1, dpi=150, bbox_inches="tight")
+    plt.close(fig1)
+    print(f"  Saved: {p1}")
+
+    # ── CNN backbones ───────────────────────────────────────────────────────
+    cnn_data, ht_data = _load_p1_cnn_aucs(phase1_dir)
+    n_b = len(P1_BACKBONES)
+    offsets2 = [-group_w / 4, group_w / 4]
+    method_colors = [C_OTHER, C_BASELINE]
+
+    fig2, ax2 = plt.subplots(figsize=(11, 5))
+    fig2.suptitle("Phase 1 — CNN backbone: standalone vs HyperTower (image only)",
+                  fontsize=FSIZE + 2, fontweight="bold")
+
+    for bi, backbone in enumerate(P1_BACKBONES):
+        for si, (lbl, data, color) in enumerate([
+            ("CNN standalone", cnn_data, method_colors[0]),
+            ("HyperTower",     ht_data,  method_colors[1]),
+        ]):
+            aucs = data.get(backbone, [])
+            if not aucs:
+                continue
+            bp = ax2.boxplot(aucs, positions=[bi + offsets2[si]], widths=box_w,
+                             patch_artist=True, manage_ticks=False,
+                             boxprops=dict(facecolor=color, alpha=0.8),
+                             medianprops=dict(color=C_MEDIAN, linewidth=2),
+                             whiskerprops=dict(color=color, linewidth=1.2),
+                             capprops=dict(color=color, linewidth=1.2),
+                             flierprops=dict(marker="o", markersize=3, alpha=0.5))
+            if bi == 0:
+                bp["boxes"][0].set_label(lbl)
+
+        if backbone in P1_PAPER_CNN:
+            mean_p, _ = P1_PAPER_CNN[backbone]
+            ax2.hlines(mean_p, bi - group_w / 2, bi + group_w / 2,
+                       colors="black", linestyles=":", linewidths=1.5,
+                       label="PAPILA paper" if bi == 0 else "_nolegend_")
+
+    ax2.set_xticks(range(n_b))
+    ax2.set_xticklabels([P1_BACKBONE_LABELS[b] for b in P1_BACKBONES], fontsize=FSIZE)
+    ax2.set_ylabel("Test AUC", fontsize=FSIZE + 1)
+    ax2.set_ylim(0.45, 1.02)
+    ax2.axhline(0.5, color="grey", linestyle="--", linewidth=0.8, alpha=0.4)
+    ax2.grid(axis="y", alpha=0.3)
+    ax2.legend(fontsize=FSIZE, loc="lower right", framealpha=0.9)
+    fig2.tight_layout()
+    p2 = out_dir / "phase1_cnn_backbones.png"
+    fig2.savefig(p2, dpi=150, bbox_inches="tight")
+    plt.close(fig2)
+    print(f"  Saved: {p2}")
+
+
+# ── Phase 2 ──────────────────────────────────────────────────────────────────
+
+PHASE2_RUNS = [
+    ("imageonly_resnet50_leaky",       "classic_test", "leaky CV"),
+    ("imageonly_resnet50_proper",      "classic_test", "baseline"),
+    ("imageonly_refugelike_proper",    "classic_test", "pretrained"),
+    ("imageonly_resnet50_gtcrop_1.1",  "classic_test", "GT crop 1.1x"),
+    ("imageonly_resnet50_gtcrop_2.5",  "classic_test", "GT crop 2.5x"),
+    ("imageonly_resnet50_unetcrop_1.1","classic_test", "UNet crop 1.1x"),
+    ("imageonly_resnet50_unetcrop_2.5","classic_test", "UNet crop 2.5x"),
+]
+PHASE2_BASELINE = "imageonly_resnet50_proper"
+PHASE2_GROUPS = {
+    "Backbone": ["imageonly_resnet50_proper", "imageonly_refugelike_proper"],
+    "GT crop": ["imageonly_resnet50_proper", "imageonly_resnet50_gtcrop_1.1", "imageonly_resnet50_gtcrop_2.5"],
+    "UNet crop": ["imageonly_resnet50_proper", "imageonly_resnet50_unetcrop_1.1", "imageonly_resnet50_unetcrop_2.5"],
+    "Data leakage": ["imageonly_resnet50_leaky", "imageonly_resnet50_proper"],
+}
+
+
+def analyze_phase2(out_dir: Path):
+    print("\n=== Phase 2 ===")
+    aucs = {}
+    for run, key, _ in PHASE2_RUNS:
+        a = load_rep_aucs(RESULTS_ROOT / "phase2" / run, key)
+        aucs[run] = a
+        print(f"  {run:40s}  AUC={np.mean(a):.3f}±{np.std(a):.3f}  n={len(a)}")
+
+    # Leakage impact
+    leaky = aucs.get("imageonly_resnet50_leaky", np.array([]))
+    proper = aucs.get("imageonly_resnet50_proper", np.array([]))
+    if len(leaky) and len(proper):
+        min_n = min(len(leaky), len(proper))
+        p = wilcoxon_p(leaky[:min_n], proper[:min_n])
+        delta = np.mean(leaky) - np.mean(proper)
+        print(f"\n  Data leakage inflates AUC by {delta:+.3f}  (Wilcoxon p={p:.4f})")
+
+    # Build display order: baseline first, then non-baseline sorted by mean AUC descending
+    base_run = PHASE2_BASELINE
+    base_label = next(lbl for r, _, lbl in PHASE2_RUNS if r == base_run)
+    others = [(r, lbl) for r, _, lbl in PHASE2_RUNS if r != base_run]
+    others.sort(key=lambda x: -np.mean(aucs[x[0]]) if len(aucs.get(x[0], [])) else float("inf"))
+    ordered = [(base_run, base_label)] + others
+
+    run_names = [r for r, _ in ordered]
+    labels    = [lbl for _, lbl in ordered]
+    base_idx  = 0
+    base_aucs = aucs[base_run]
+
+    fig, ax = plt.subplots(figsize=(14, 6))
+    fig.suptitle("Phase 2 — ResNet50: Backbone & Preprocessing Comparison", fontsize=12, fontweight="bold")
+
+    data    = [aucs[r] for r in run_names]
+    colors  = ["#dd8452" if r == base_run else "#4c72b0" for r in run_names]
+    x       = np.arange(len(run_names))
+
+    bp = ax.boxplot(data, vert=True, patch_artist=True, positions=x,
+                    widths=0.3, showfliers=True,
+                    flierprops=dict(marker="o", markersize=3, alpha=0.5),
+                    medianprops=dict(color="#c44e52", linewidth=2))
+    for patch, color in zip(bp["boxes"], colors):
+        patch.set_facecolor(color)
+        patch.set_alpha(0.8)
+
+    ax.set_xlim(-0.5, len(run_names) - 0.5)
+    ax.set_ylabel("Test AUC", fontsize=11)
+    ax.axhline(np.median(base_aucs), color="#dd8452", linewidth=1,
+               linestyle="--", alpha=0.5, label="Baseline median")
+    ax.legend(fontsize=9)
+    ax.grid(axis="y", alpha=0.3)
+
+    # Build x-tick labels with p-value on a second line underneath
+    tick_labels = []
+    for i, run in enumerate(run_names):
+        if run == base_run:
+            tick_labels.append(labels[i])
+            continue
+        a = aucs[run]
+        min_n = min(len(a), len(base_aucs))
+        p = wilcoxon_p(a[:min_n], base_aucs[:min_n])
+        p_str = f"p={p:.3f}" if not np.isnan(p) else "p=n/a"
+        tick_labels.append(f"{labels[i]}\n{p_str}")
+
+    ax.set_xticks(x)
+    ax.set_xticklabels(tick_labels, fontsize=10)
+
+    fig.tight_layout()
+    path = out_dir / "phase2_analysis.png"
+    fig.savefig(path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+# ── Phase 3 ──────────────────────────────────────────────────────────────────
+
+PHASE3_GROUPS = {
+    "Loss function": {
+        "baseline":    "Baseline (BCE fused)",
+        "loss_all":    "All losses",
+        "loss_bcd_p03":"BCD p=0.3",
+        "loss_bcd_p07":"BCD p=0.7",
+    },
+    "SE attention": {
+        "baseline":      "Baseline",
+        "se_img_tower":  "SE img tower",
+        "se_cd_tower":   "SE cd tower",
+        "se_bridge":     "SE bridge",
+        "se_all":        "SE all",
+    },
+    "IOP correction": {
+        "baseline":          "Baseline (none)",
+        "iop_ratio":         "Ratio",
+        "iop_ratio_drop_raw":"Ratio + drop raw",
+        "iop_ols":           "OLS",
+        "iop_lad":           "LAD",
+        "iop_multi":         "Multi",
+    },
+    "Feature ablation": {
+        "baseline":        "Baseline (all)",
+        "excl_iop":        "Excl IOP",
+        "excl_age":        "Excl age",
+        "excl_axial_length":"Excl axial length",
+        "excl_refractive": "Excl refractive",
+    },
+    "Network dims": {
+        "baseline":       "Baseline",
+        "cd_hidden_64":   "CD hidden=64",
+        "cd_hidden_256":  "CD hidden=256",
+        "fusion_dim_128": "Fusion dim=128",
+        "fusion_dim_512": "Fusion dim=512",
+    },
+    "Dropout": {
+        "baseline":         "Baseline (0.5)",
+        "bridge_dropout_03":"Bridge drop=0.3",
+        "bridge_dropout_07":"Bridge drop=0.7",
+        "cd_dropout_03":    "CD drop=0.3",
+    },
+    "Backbone freezing": {
+        "baseline":  "Baseline (75%)",
+        "freeze_25": "Freeze 25%",
+        "freeze_50": "Freeze 50%",
+    },
+    "Warmup": {
+        "baseline":             "Baseline (cd40+twr3+fus3)",
+        "warmup_no_cd":         "No CD warmup",
+        "warmup_tower5_fused5": "Tower5+Fused5",
+    },
+    "Sampling": {
+        "baseline":           "Baseline",
+        "balanced_sampling":  "Balanced sampling",
+    },
+    "Epoch length": {
+        "epochs_1":  "1 epoch",
+        "epochs_5":  "5 epochs",
+        "epochs_10": "10 epochs",
+        "epochs_20": "20 epochs",
+        "epochs_30": "30 epochs (baseline)",
+        "epochs_50": "50 epochs",
+    },
+    "Learning rate": {
+        "baseline": "Baseline (1e-4)",
+        "lr_3e4":   "3e-4",
+        "lr_1e3":   "1e-3",
+        "lr_1e5":   "1e-5",
+    },
+}
+PHASE3_BASELINE = "baseline"
+
+
+def analyze_phase3(out_dir: Path):
+    print("\n=== Phase 3 ===")
+    all_runs = set()
+    for group in PHASE3_GROUPS.values():
+        all_runs.update(group.keys())
+    aucs = {}
+    for run in all_runs:
+        a = load_rep_aucs(RESULTS_ROOT / "phase3" / run, "classic_test")
+        aucs[run] = a
+    base_aucs = aucs[PHASE3_BASELINE]
+    print(f"  Baseline AUC: {np.mean(base_aucs):.3f}±{np.std(base_aucs):.3f}")
+
+    # Build display order: baseline first, then each group (non-baseline, sorted desc)
+    GAP = 1.2   # extra space between groups
+    pos = 0.0
+    positions, box_data, tick_labels, colors, is_sig = [], [], [], [], []
+    group_spans = []  # (x_mid, group_name) for title annotations
+
+    # Baseline box
+    positions.append(pos)
+    box_data.append(base_aucs)
+    tick_labels.append("baseline")
+    colors.append(C_BASELINE)
+    is_sig.append(False)
+    pos += 1 + GAP
+
+    def _group_max_median(group_runs):
+        vals = [np.median(aucs[r]) for r in group_runs if r != PHASE3_BASELINE and len(aucs.get(r, []))]
+        return max(vals) if vals else 0.0
+
+    sorted_groups = sorted(PHASE3_GROUPS.items(), key=lambda x: -_group_max_median(x[1]))
+
+    for shade_idx, (group_name, group_runs) in enumerate(sorted_groups):
+        non_base = [(r, lbl) for r, lbl in group_runs.items() if r != PHASE3_BASELINE]
+        non_base.sort(key=lambda x: -np.mean(aucs[x[0]]) if len(aucs.get(x[0], [])) else float("inf"))
+
+        group_start = pos
+        for run, lbl in non_base:
+            a = aucs.get(run, np.array([]))
+            positions.append(pos)
+            box_data.append(a)
+            # p-value label under name
+            min_n = min(len(a), len(base_aucs))
+            p = wilcoxon_p(a[:min_n], base_aucs[:min_n]) if min_n >= 5 else float("nan")
+            sig = not np.isnan(p) and p < 0.05
+            if sig:
+                tick_labels.append(f"* {lbl}\np={p:.3f}")
+            else:
+                tick_labels.append(lbl)
+            colors.append(C_OTHER)
+            is_sig.append(sig)
+            pos += 1
+        group_spans.append(((group_start + pos - 1) / 2, group_name, group_start, pos - 1, shade_idx))
+        pos += GAP
+
+    fig, ax = plt.subplots(figsize=(9, 22))
+    fig.suptitle("Phase 3 — Clinical Fusion Ablations (Single-Eye)",
+                 fontsize=FSIZE + 3, fontweight="bold")
+    fig.subplots_adjust(top=0.97, left=0.38)
+
+    bp = ax.boxplot(box_data, vert=False, patch_artist=True, positions=positions,
+                    widths=0.5, showfliers=True,
+                    flierprops=dict(marker="o", markersize=3, alpha=0.5),
+                    medianprops=dict(color=C_MEDIAN, linewidth=2),
+                    manage_ticks=False)
+    for patch, color in zip(bp["boxes"], colors):
+        patch.set_facecolor(color)
+        patch.set_alpha(0.8)
+
+    # Alternating shaded group backgrounds
+    for x_mid, gname, g_start, g_end, shade_idx in group_spans:
+        if shade_idx % 2 == 0:
+            ax.axhspan(g_start - 0.5, g_end + 0.5, color="gray", alpha=0.07, zorder=0)
+        # Group title to the left of the y-tick labels
+        ax.text(-0.42, x_mid, gname, transform=ax.get_yaxis_transform(),
+                ha="right", va="center", fontsize=FSIZE - 1, fontweight="bold", color="#444444")
+
+    ax.axvline(np.median(base_aucs), color=C_BASELINE, linewidth=1,
+               linestyle="--", alpha=0.5, label="Baseline median")
+    ax.set_yticks(positions)
+    ax.set_yticklabels(tick_labels, fontsize=FSIZE - 1)
+    for tick_lbl, sig in zip(ax.get_yticklabels(), is_sig):
+        if sig:
+            tick_lbl.set_fontweight("bold")
+    ax.set_xlabel("Test AUC", fontsize=FSIZE + 1)
+    ax.set_xlim(0.65, None)
+    ax.set_ylim(-0.7, pos - GAP + 0.7)
+    ax.invert_yaxis()  # baseline at top
+    ax.grid(axis="x", alpha=0.3)
+    ax.legend(fontsize=FSIZE, loc="lower right")
+    path = out_dir / "phase3_single_mode_ablations.png"
+    fig.savefig(path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+    # Print top winners vs baseline
+    print("\n  Top movers vs baseline (Wilcoxon, uncorrected):")
+    deltas = []
+    for run in aucs:
+        if run == PHASE3_BASELINE:
+            continue
+        a = aucs[run]
+        min_n = min(len(a), len(base_aucs))
+        if min_n < 5:
+            continue
+        delta = np.mean(a) - np.mean(base_aucs)
+        p = wilcoxon_p(a[:min_n], base_aucs[:min_n])
+        deltas.append((run, delta, p))
+    deltas.sort(key=lambda x: -x[1])
+    for run, delta, p in deltas[:8]:
+        print(f"    {run:30s}  {delta:+.3f}  p={p:.4f} {stars(p)}")
+
+    # Pairwise table — IOP correction group (FDR-corrected Wilcoxon p-values)
+    iop_runs   = list(PHASE3_GROUPS["IOP correction"].keys())
+    iop_labels = list(PHASE3_GROUPS["IOP correction"].values())
+    _, corr = paired_matrix(iop_runs, aucs)
+    reports_dir = out_dir / "reports"
+    reports_dir.mkdir(exist_ok=True)
+    import csv
+    path2 = reports_dir / "phase3_iop_pairwise.csv"
+    with open(path2, "w", newline="") as f:
+        w = csv.writer(f)
+        w.writerow([""] + iop_labels)
+        for i, row_lbl in enumerate(iop_labels):
+            cells = [row_lbl]
+            for j in range(len(iop_labels)):
+                p = corr[i, j]
+                cells.append(f"{p:.4f} {stars(p)}" if not np.isnan(p) else "—")
+            w.writerow(cells)
+    print(f"  Saved: {path2}")
+
+
+# ── Phase 4 ──────────────────────────────────────────────────────────────────
+
+PHASE4_RUNS = {
+    "single":             ("classic_test",  "Single-eye\n(baseline)"),
+    "ensemble":           ("ensemble_test", "Ensemble\n(indep OD+OS)"),
+    "bilateral":          ("bilat_test",    "BilateralHT\n(shared+concat)"),
+    "siamese":            ("bilat_test",    "SiameseHT\n(mean+delta)"),
+    "bilateral_loss_all": ("bilat_test",    "BilateralHT\nall-losses"),
+    "siamese_loss_all":   ("bilat_test",    "SiameseHT\nall-losses"),
+}
+PHASE4_BASELINE = "single"
+
+
+def analyze_phase4(out_dir: Path):
+    print("\n=== Phase 4 ===")
+    aucs = {}
+    for run, (key, _) in PHASE4_RUNS.items():
+        a = load_rep_aucs(RESULTS_ROOT / "phase4" / run, key)
+        aucs[run] = a
+        print(f"  {run:25s}  AUC={np.mean(a):.3f}±{np.std(a):.3f}  n={len(a)}")
+
+    base_run  = PHASE4_BASELINE
+    base_aucs = aucs[base_run]
+    others = [(r, PHASE4_RUNS[r][1]) for r in PHASE4_RUNS if r != base_run]
+    others.sort(key=lambda x: -np.mean(aucs[x[0]]) if len(aucs.get(x[0], [])) else float("inf"))
+    ordered   = [(base_run, PHASE4_RUNS[base_run][1])] + others
+    run_keys  = [r for r, _ in ordered]
+    labels    = [lbl for _, lbl in ordered]
+    data_list = [aucs[r] for r in run_keys]
+    aucs_by_label = {lbl: aucs[r] for r, lbl in ordered}
+
+    fig, ax = plt.subplots(figsize=(11, 5))
+    fig.suptitle("Phase 4 — Bilateral Architecture Comparison (Image Only)",
+                 fontsize=FSIZE + 2, fontweight="bold")
+    boxplot_panel(ax, data_list, labels, base_idx=0,
+                  base_aucs=base_aucs, all_aucs_by_label=aucs_by_label)
+    ax.set_ylim(0.75, None)
+
+    fig.tight_layout()
+    path = out_dir / "phase4_analysis.png"
+    fig.savefig(path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+# ── Phase 5 ──────────────────────────────────────────────────────────────────
+
+PHASE5_RUNS = {
+    # "single_fused":        ("classic_test",   "Single-eye (baseline)"),
+    "ensemble_fused":      ("ensemble_test",  "Ensemble (baseline)"),
+    "bilateral_fused":     ("bilat_test",     "BilateralHT"),
+    "siamese_fused":       ("bilat_test",     "SiameseHT"),
+    # "ensemble_fused_head": ("ensemble_test",  "Ensemble\n+clinical+head"),
+    "logit_mlp_head": ("ensemble_test",  "Ensemble\n+Fusion head"),
+}
+PHASE5_BASELINE = "ensemble_fused"
+
+
+def analyze_phase5(out_dir: Path):
+    print("\n=== Phase 5 ===")
+    aucs = {}
+    for run, (key, _) in PHASE5_RUNS.items():
+        a = load_rep_aucs(RESULTS_ROOT / "phase5" / run, key)
+        aucs[run] = a
+        print(f"  {run:25s}  AUC={np.mean(a):.3f}±{np.std(a):.3f}  n={len(a)}")
+
+    base_run  = PHASE5_BASELINE
+    base_aucs = aucs[base_run]
+    others = [(r, PHASE5_RUNS[r][1]) for r in PHASE5_RUNS if r != base_run]
+    others.sort(key=lambda x: -np.mean(aucs[x[0]]) if len(aucs.get(x[0], [])) else float("inf"))
+    ordered   = [(base_run, PHASE5_RUNS[base_run][1])] + others
+    run_keys  = [r for r, _ in ordered]
+    labels    = [lbl for _, lbl in ordered]
+    data_list = [aucs[r] for r in run_keys]
+    aucs_by_label = {lbl: aucs[r] for r, lbl in ordered}
+
+    fig, ax = plt.subplots(figsize=(11, 5))
+    fig.suptitle("Phase 5 — Full HyperTower: Bilateral + Clinical",
+                 fontsize=FSIZE + 2, fontweight="bold")
+    boxplot_panel(ax, data_list, labels, base_idx=0,
+                  base_aucs=base_aucs, all_aucs_by_label=aucs_by_label)
+    ax.set_ylim(0.78, None)
+
+    fig.tight_layout()
+    path = out_dir / "phase5_analysis.png"
+    fig.savefig(path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+# ── Cross-phase summary ───────────────────────────────────────────────────────
+
+def analyze_cross_phase(out_dir: Path):
+    """Single figure tracing the best model from each phase."""
+    print("\n=== Cross-phase progression ===")
+    trajectory = [
+        ("Phase 2\nresnet50 proper",   "phase2", "imageonly_resnet50_proper",   "classic_test"),
+        ("Phase 2\nrefugelike proper", "phase2", "imageonly_refugelike_proper",  "classic_test"),
+        ("Phase 3\n+IOP ratio\n+drop raw", "phase3", "iop_ratio_drop_raw",     "classic_test"),
+        ("Phase 4\nensemble\n(image only)", "phase4", "ensemble",               "ensemble_test"),
+        ("Phase 5\nensemble\n+clinical",    "phase5", "ensemble_fused",         "ensemble_test"),
+        ("Phase 5\nensemble\n+clinical+head","phase5","ensemble_fused_head",    "ensemble_test"),
+    ]
+
+    labels, means, stds, all_aucs = [], [], [], []
+    for lbl, phase, run, key in trajectory:
+        a = load_rep_aucs(RESULTS_ROOT / phase / run, key)
+        labels.append(lbl)
+        means.append(np.mean(a) if len(a) else np.nan)
+        stds.append(np.std(a) if len(a) else np.nan)
+        all_aucs.append(a)
+        print(f"  {lbl.replace(chr(10),' '):35s}  AUC={means[-1]:.3f}±{stds[-1]:.3f}  n={len(a)}")
+
+    fig, ax = plt.subplots(figsize=(10, 4))
+    x = np.arange(len(labels))
+    ax.errorbar(x, means, yerr=stds, fmt="o-", linewidth=2, markersize=7,
+                capsize=4, color="#4c72b0")
+    ax.set_xticks(x)
+    ax.set_xticklabels(labels, fontsize=8)
+    ax.set_ylabel("Test AUC (10-rep mean ± std)")
+    ax.set_title("HyperTower — Model Progression Across Phases", fontsize=12, fontweight="bold")
+    ax.set_ylim(0.75, 0.95)
+    ax.axhline(means[0], color="gray", linewidth=1, linestyle=":", alpha=0.5, label="Phase 2 baseline")
+    ax.legend(fontsize=8)
+    ax.grid(axis="y", alpha=0.3)
+
+    fig.tight_layout()
+    path = out_dir / "cross_phase_progression.png"
+    fig.savefig(path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+# ── Main ─────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--phase", type=int, choices=[1, 2, 3, 4, 5],
+                    help="Run only this phase (default: all)")
+    ap.add_argument("--out", type=Path,
+                    default=Path(__file__).resolve().parents[3] / "v3" / "figures",
+                    help="Output directory for figures")
+    args = ap.parse_args()
+
+    args.out.mkdir(parents=True, exist_ok=True)
+
+    run_all = args.phase is None
+    if run_all or args.phase == 1:
+        analyze_phase1(args.out)
+    if run_all or args.phase == 2:
+        analyze_phase2(args.out)
+    if run_all or args.phase == 3:
+        analyze_phase3(args.out)
+    if run_all or args.phase == 4:
+        analyze_phase4(args.out)
+    if run_all or args.phase == 5:
+        analyze_phase5(args.out)
+    if run_all:
+        analyze_cross_phase(args.out)
+
+
+if __name__ == "__main__":
+    main()

v3/scripts/output_analysis/explainability/comparison_panel_phase5.py

@@ -0,0 +1,264 @@
+"""
+Phase 5 comparison panel — ROC curves + fusion event summaries.
+
+Layout:
+  Top row    (1 × 3)  — ROC curves: Single HyperTower | Bilateral Ensemble | Fused Head
+  Bottom rows (2 × 1)  — Fusion event summary (full-width) for Ensemble then Fused Head
+                          (Single mode has fused-only bridge; no meaningful fusion events)
+
+All data derived from predictions_test.csv — no checkpoints required.
+
+Usage:
+    python -m v3.scripts.output_analysis.explainability.comparison_panel_phase5
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.patches as mpatches
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from sklearn.metrics import roc_auc_score, roc_curve
+
+REPO_ROOT    = Path(__file__).resolve().parents[4]
+RESULTS_ROOT = REPO_ROOT / "v3" / "results"
+FIGURES_ROOT = REPO_ROOT / "v3" / "figures" / "explainability"
+CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
+
+RUNS = [
+    {"label": "Single HyperTower", "run": "phase5/single_fused",
+     "tower_path": "binary/single",   "is_single": False},
+    {"label": "Bilateral Ensemble", "run": "phase5/ensemble_fused",
+     "tower_path": "binary/ensemble", "is_single": False},
+    {"label": "Fused Head",         "run": "phase5/logit_mlp_head",
+     "tower_path": "binary/ensemble", "is_single": False},
+]
+
+# Event taxonomy (img=image tower, md=clinical tower)
+_EVENT_KEYS = [
+    "full_correction", "img_assist", "md_assist",
+    "full_error",      "img_drag",   "md_drag",
+    "concordant_correct", "concordant_wrong",
+]
+_EVENT_COLORS = [
+    "#2ca02c", "#98df8a", "#b5cf6b",   # positive
+    "#d62728", "#ff9896", "#ffbb78",   # negative
+    "#aec7e8", "#c5b0d5",              # concordant
+]
+_POSITIVE_KEYS = _EVENT_KEYS[:3]
+_NEGATIVE_KEYS = _EVENT_KEYS[3:6]
+_DISAGREE_KEYS = _POSITIVE_KEYS + _NEGATIVE_KEYS   # exclude concordant
+
+
+# ── Data loading ──────────────────────────────────────────────────────────────
+
+def load_pooled(run: str, tower_path: str) -> pd.DataFrame:
+    run_dir = RESULTS_ROOT / run
+    rows = []
+    for rep in sorted(run_dir.glob("rep*")):
+        tm = rep / tower_path
+        if not tm.exists():
+            continue
+        for fold in sorted(tm.glob("fold[0-9]")):
+            csv = fold / "predictions_test.csv"
+            if csv.exists():
+                df = pd.read_csv(csv)
+                df["rep"]  = rep.name
+                df["fold"] = fold.name
+                rows.append(df)
+    if not rows:
+        raise FileNotFoundError(f"No predictions found under {run_dir}/{tower_path}")
+    return pd.concat(rows, ignore_index=True)
+
+
+def classify_events(df: pd.DataFrame) -> pd.DataFrame:
+    """Add event_type column based on pred_fused/pred_img/pred_md vs y_true."""
+    df = df.copy()
+    y  = df["y_true"].values
+    pf = df["pred_fused"].values
+    pi = df["pred_img"].values
+    pm = df["pred_md"].values
+
+    fused_ok = pf == y
+    img_ok   = pi == y
+    md_ok    = pm == y
+
+    def _classify(fo, io, mo):
+        if fo and io and mo:   return "concordant_correct"
+        if not fo and not io and not mo: return "concordant_wrong"
+        if fo and not io and not mo:     return "full_correction"
+        if fo and io and not mo:         return "img_assist"
+        if fo and not io and mo:         return "md_assist"
+        if not fo and io and mo:         return "full_error"
+        if not fo and not io and mo:     return "img_drag"
+        if not fo and io and not mo:     return "md_drag"
+        return "other"
+
+    df["event_type"]  = [_classify(fo, io, mo)
+                         for fo, io, mo in zip(fused_ok, img_ok, md_ok)]
+    # conf_delta: fused prob minus average of img/md
+    df["conf_fused"]  = df["prob_fused_c1"]
+    df["conf_img"]    = df["prob_img_c1"]
+    df["conf_md"]     = df["prob_md_c1"]
+    df["conf_delta"]  = df["conf_fused"] - 0.5 * (df["conf_img"] + df["conf_md"])
+    return df
+
+
+# ── ROC panel ─────────────────────────────────────────────────────────────────
+
+def _draw_roc(ax, df: pd.DataFrame, label: str, color: str) -> None:
+    """Draw per-fold ROC curves (faint) + mean ROC (bold) on ax."""
+    fold_aucs = []
+    for (rep, fold), grp in df.groupby(["rep", "fold"]):
+        if grp["y_true"].nunique() < 2:
+            continue
+        fpr, tpr, _ = roc_curve(grp["y_true"], grp["prob_fused_c1"])
+        ax.plot(fpr, tpr, color=color, alpha=0.12, lw=0.8)
+        fold_aucs.append(roc_auc_score(grp["y_true"], grp["prob_fused_c1"]))
+
+    # Mean ROC via interpolation
+    mean_fpr = np.linspace(0, 1, 200)
+    tprs = []
+    for (rep, fold), grp in df.groupby(["rep", "fold"]):
+        if grp["y_true"].nunique() < 2:
+            continue
+        fpr, tpr, _ = roc_curve(grp["y_true"], grp["prob_fused_c1"])
+        tprs.append(np.interp(mean_fpr, fpr, tpr))
+    mean_tpr = np.mean(tprs, axis=0)
+    mean_auc = np.mean(fold_aucs)
+    std_auc  = np.std(fold_aucs)
+    ax.plot(mean_fpr, mean_tpr, color=color, lw=2.2,
+            label=f"Mean AUC = {mean_auc:.3f} ± {std_auc:.3f}")
+    ax.fill_between(mean_fpr,
+                    np.percentile(tprs, 25, axis=0),
+                    np.percentile(tprs, 75, axis=0),
+                    color=color, alpha=0.12)
+    ax.plot([0, 1], [0, 1], "k--", lw=0.7, alpha=0.5)
+    ax.set_xlim(-0.02, 1.02); ax.set_ylim(-0.02, 1.02)
+    ax.set_xlabel("False Positive Rate", fontsize=9)
+    ax.set_ylabel("True Positive Rate", fontsize=9)
+    ax.set_title(label, fontsize=10, fontweight="bold")
+    ax.legend(fontsize=8, loc="lower right")
+    ax.grid(alpha=0.25)
+
+
+# ── Fusion summary panel ──────────────────────────────────────────────────────
+
+def _draw_fusion_summary(axes_row, df: pd.DataFrame, label: str) -> None:
+    """Draw 3-panel fusion summary (disagreement events only) on axes_row (list of 3 axes)."""
+    event_color = dict(zip(_EVENT_KEYS, _EVENT_COLORS))
+    event_labels = {
+        "full_correction": "Full correction\n(both wrong → right)",
+        "img_assist":      "Img assist\n(img✓ md✗ → right)",
+        "md_assist":       "MD assist\n(md✓ img✗ → right)",
+        "full_error":      "Full error\n(both right → wrong)",
+        "img_drag":        "Img drag\n(img✗ md✓ → wrong)",
+        "md_drag":         "MD drag\n(md✗ img✓ → wrong)",
+    }
+
+    # Only count disagreement events (exclude concordant)
+    counts = {k: (df["event_type"] == k).sum() for k in _DISAGREE_KEYS}
+
+    # Panel 0: totals bar (positive vs negative)
+    ax = axes_row[0]
+    for bar_x, keys in ((0, _POSITIVE_KEYS), (1, _NEGATIVE_KEYS)):
+        bot = 0
+        for k in keys:
+            c = int(counts[k])
+            ax.bar(bar_x, c, bottom=bot, color=event_color[k], width=0.5)
+            if c > 0:
+                ax.text(bar_x, bot + c / 2, str(c), ha="center", va="center",
+                        fontsize=8, fontweight="bold")
+            bot += c
+    ax.set_xticks([0, 1]); ax.set_xticklabels(["Positive\nevents", "Negative\nevents"])
+    ax.set_ylabel("Count (all folds)")
+    patches = [mpatches.Patch(color=event_color[k], label=event_labels[k].split("\n")[0])
+               for k in _DISAGREE_KEYS if counts[k] > 0]
+    ax.legend(handles=patches, fontsize=6, loc="upper right")
+    ax.set_title(f"{label}\nDisagreement event totals", fontsize=9)
+
+    # Panel 1: per-fold stacked bar (disagreement events only)
+    ax = axes_row[1]
+    fold_groups = sorted(df.groupby(["rep", "fold"]), key=lambda x: x[0])
+    x = np.arange(len(fold_groups))
+    pos_bot = np.zeros(len(fold_groups))
+    neg_bot = np.zeros(len(fold_groups))
+    for k, color in zip(_POSITIVE_KEYS, _EVENT_COLORS[:3]):
+        vals = np.array([(g["event_type"] == k).sum() for _, g in fold_groups], dtype=float)
+        ax.bar(x, vals, bottom=pos_bot, color=color, width=0.6)
+        pos_bot += vals
+    for k, color in zip(_NEGATIVE_KEYS, _EVENT_COLORS[3:6]):
+        vals = np.array([(g["event_type"] == k).sum() for _, g in fold_groups], dtype=float)
+        ax.bar(x + 0.65, vals, bottom=neg_bot, color=color, width=0.6)
+        neg_bot += vals
+    ax.set_xticks([])
+    ax.set_xlabel("Fold", fontsize=8)
+    ax.set_ylabel("Count"); ax.set_title("Per-fold breakdown\n(left=positive, right=negative)", fontsize=9)
+
+    # Panel 2: img vs md confidence scatter (disagreement events only)
+    ax = axes_row[2]
+    for k in _DISAGREE_KEYS:
+        sub = df[df["event_type"] == k]
+        if len(sub) == 0:
+            continue
+        ax.scatter(sub["conf_img"], sub["conf_md"], c=event_color[k],
+                   alpha=0.65, s=30, edgecolors="none",
+                   label=event_labels[k].split("\n")[0])
+    ax.plot([0, 1], [0, 1], "k--", lw=0.5, alpha=0.4)
+    ax.set_xlabel("P(Glaucoma) — Image head"); ax.set_ylabel("P(Glaucoma) — MD head")
+    ax.legend(fontsize=5.5, loc="lower right")
+    ax.set_title("Tower confidence space\n(disagreement events only)", fontsize=9)
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    ROC_COLORS = ["#4c72b0", "#dd8452", "#55a868"]
+
+    print("Loading predictions ...")
+    datasets = []
+    for cfg, color in zip(RUNS, ROC_COLORS):
+        df = load_pooled(cfg["run"], cfg["tower_path"])
+        df = classify_events(df)
+        datasets.append((cfg, df, color))
+
+    fusion_runs = [(cfg, df, color) for cfg, df, color in datasets]
+
+    # ── Layout ────────────────────────────────────────────────────────────────
+    # Row 0:        3 ROC axes
+    # Rows 1,2:     2 fusion summary strips (5 axes each, spanning full width)
+    n_fusion = len(fusion_runs)
+    fig = plt.figure(figsize=(20, 6 + 4.5 * n_fusion))
+    gs = fig.add_gridspec(
+        1 + n_fusion, 1,
+        height_ratios=[5] + [4.5] * n_fusion,
+        hspace=0.35,
+    )
+
+    # ROC row — subdivide into 3
+    roc_gs = gs[0].subgridspec(1, 3, wspace=0.28)
+    for i, (cfg, df, color) in enumerate(datasets):
+        ax = fig.add_subplot(roc_gs[i])
+        _draw_roc(ax, df, cfg["label"], color)
+
+    # Fusion rows (3 panels each)
+    for fi, (cfg, df, color) in enumerate(fusion_runs):
+        fus_gs = gs[1 + fi].subgridspec(1, 3, wspace=0.32)
+        axes_row = [fig.add_subplot(fus_gs[j]) for j in range(3)]
+        _draw_fusion_summary(axes_row, df, cfg["label"])
+
+    fig.suptitle("Phase 5 — Model Comparison: ROC Curves & Fusion Event Analysis",
+                 fontsize=13, fontweight="bold", y=1.01)
+
+    out = FIGURES_ROOT / "comparison_panel_phase5.png"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(out, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"Saved: {out}")
+
+
+if __name__ == "__main__":
+    main()

v3/scripts/output_analysis/explainability/confidence_strips.py

@@ -0,0 +1,615 @@
+"""
+Phase 5 explainability — confidence strips and head comparison.
+
+Works from saved prediction CSVs. If patient_id column is present (requires
+a re-run after the v3_hypertower.py update), points are colored by VFI
+severity group. Otherwise falls back to a single color per true class.
+
+VFI severity groups (VF_MD from clinical data):
+  Early    VF_MD > -6
+  Moderate VF_MD -6 to -12
+  Severe   VF_MD < -12
+
+Produces (all in figures/explainability/):
+  confidence_strips.png   — vertical strip: P(glaucoma) by true class, VFI colored
+  head_comparison.png     — fused vs img vs md distributions side by side
+
+Usage:
+    python -m v3.scripts.output_analysis.explainability.confidence_strips
+    python -m v3.scripts.output_analysis.explainability.confidence_strips \
+        --run phase5/logit_mlp_head --clinical-dir Papila/ClinicalData
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+import matplotlib
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import matplotlib.patches as mpatches
+import numpy as np
+import pandas as pd
+
+REPO_ROOT = Path(__file__).resolve().parents[4]
+RESULTS_ROOT = REPO_ROOT / "v3" / "results"
+FIGURES_ROOT = REPO_ROOT / "v3" / "figures"
+CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
+
+FONT = "DejaVu Sans"
+
+# Colour palette
+C_NORMAL = "#78909C"  # blue-grey   — healthy controls (no VFI staging)
+C_EARLY = "#29B6F6"  # sky-blue    — glaucoma, early VFI loss
+C_MODERATE = "#FFB300"  # amber       — glaucoma, moderate VFI loss
+C_SEVERE = "#E53935"  # vivid red   — glaucoma, severe VFI loss
+C_UNKNOWN = "#BDBDBD"  # light grey  — glaucoma, VFI not recorded
+
+SEV_LABELS = {
+    "normal": "Normal",
+    "early": "Glaucoma — early (VF_MD > −6)",
+    "moderate": "Glaucoma — moderate (−12 to −6)",
+    "severe": "Glaucoma — severe (VF_MD < −12)",
+    "unknown": "Glaucoma — VF_MD not recorded",
+}
+SEV_COLORS = {
+    "normal": C_NORMAL,
+    "early": C_EARLY,
+    "moderate": C_MODERATE,
+    "severe": C_SEVERE,
+    "unknown": C_UNKNOWN,
+}
+
+HEAD_COLORS = {"fused": "#d4a017", "img": "#4e8d3a", "md": "#4c72b0"}
+HEAD_LABELS = {
+    "fused": "Fused head",
+    "img": "Image-only head",
+    "md": "Clinical-only head",
+}
+
+
+# ── VFI data ──────────────────────────────────────────────────────────────────
+
+
+def load_vfi(clinical_dir: Path) -> pd.DataFrame:
+    """Return DataFrame with columns [patient_id (int), vf_md (float), severity (str)].
+    Only includes patients in the binary study (Diagnosis 0=Normal, 1=Glaucoma).
+    """
+    od = pd.read_excel(clinical_dir / "patient_data_od.xlsx", header=1)
+    os_ = pd.read_excel(clinical_dir / "patient_data_os.xlsx", header=1)
+
+    def _clean(df, eye):
+        df = df.copy()
+        if "Patient ID" not in df.columns and "ID" in df.columns:
+            df.rename(columns={"ID": "Patient ID"}, inplace=True)
+        df["Patient ID"] = (
+            df["Patient ID"].astype(str).str.extract(r"(\d+)")[0].astype(int)
+        )
+        df["Diagnosis"] = pd.to_numeric(df["Diagnosis"], errors="coerce")
+        df["VF_MD"] = pd.to_numeric(df["VF_MD"], errors="coerce")
+        # PAPILA: 0=Normal, 1=Glaucoma, 2=Suspect — keep only binary patients
+        df = df[df["Diagnosis"].isin([0, 1])].copy()
+        df["eye"] = eye
+        return df[["Patient ID", "Diagnosis", "VF_MD", "eye"]]
+
+    combined = pd.concat([_clean(od, "OD"), _clean(os_, "OS")], ignore_index=True)
+
+    # Per patient: modal diagnosis, worst (most negative) VF_MD across eyes
+    diag = (
+        combined.groupby("Patient ID")["Diagnosis"]
+        .agg(lambda x: x.mode().iloc[0])
+        .reset_index()
+    )
+    vf = combined.groupby("Patient ID")["VF_MD"].min().reset_index()
+    worst = diag.merge(vf, on="Patient ID").rename(
+        columns={"Patient ID": "patient_id", "VF_MD": "vf_md", "Diagnosis": "diagnosis"}
+    )
+
+    def _severity(row):
+        if int(row["diagnosis"]) == 0:
+            return "normal"  # healthy control — no VFI staging
+        v = row["vf_md"]
+        if pd.isna(v):
+            return "unknown"  # glaucoma, no VFI recorded
+        if v > -6:
+            return "early"
+        if v > -12:
+            return "moderate"
+        return "severe"
+
+    worst["severity"] = worst.apply(_severity, axis=1)
+    return worst
+
+
+# ── Prediction loading ────────────────────────────────────────────────────────
+
+
+def load_all_predictions(
+    run_dir: Path, tower_path: str = "binary/ensemble"
+) -> pd.DataFrame:
+    """Pool predictions_test.csv across all reps and folds."""
+    rows = []
+    for rep_dir in sorted(run_dir.glob("rep*")):
+        tm_dir = rep_dir / tower_path
+        if not tm_dir.exists():
+            continue
+        for fold_dir in sorted(tm_dir.glob("fold[0-9]")):
+            csv_path = fold_dir / "predictions_test.csv"
+            if not csv_path.exists():
+                continue
+            df = pd.read_csv(csv_path)
+            df["rep"] = rep_dir.name
+            df["fold"] = fold_dir.name
+            rows.append(df)
+    if not rows:
+        raise FileNotFoundError(f"No predictions_test.csv found under {run_dir}")
+    return pd.concat(rows, ignore_index=True)
+
+
+# ── Figure 1: Confidence strips (vertical) ───────────────────────────────────
+
+
+def make_confidence_strips(
+    df: pd.DataFrame, vfi: pd.DataFrame | None, out_path: Path
+) -> None:
+    """
+    Vertical strip plot: x = true class, y = P(glaucoma).
+    Points colored by VFI severity if patient_id column available, else uniform.
+    """
+    rng = np.random.default_rng(42)
+    has_vfi = (
+        vfi is not None
+        and "patient_id" in df.columns
+        and df["patient_id"].notna().any()
+    )
+
+    if has_vfi:
+        df = df.copy()
+        df["patient_id"] = pd.to_numeric(df["patient_id"], errors="coerce").astype(
+            "Int64"
+        )
+        vfi_merge = vfi.copy()
+        vfi_merge["patient_id"] = vfi_merge["patient_id"].astype("Int64")
+        df = df.merge(
+            vfi_merge[["patient_id", "severity"]], on="patient_id", how="left"
+        )
+        df["severity"] = df["severity"].fillna("unknown")
+    else:
+        df["severity"] = "unknown"
+
+    fig, ax = plt.subplots(figsize=(6, 7))
+    fig.patch.set_facecolor("#e8e8e8")
+    ax.set_facecolor("#e8e8e8")
+
+    x_pos = {0: 0.0, 1: 1.0}
+    jitter_scale = 0.18
+
+    # Draw in severity order so severe is on top
+    sev_order = ["normal", "unknown", "early", "moderate", "severe"]
+    sev_alpha = {
+        "normal": 0.40,
+        "early": 0.55,
+        "moderate": 0.70,
+        "severe": 0.85,
+        "unknown": 0.35,
+    }
+    sev_size = {"normal": 6, "early": 8, "moderate": 10, "severe": 12, "unknown": 6}
+
+    for sev in sev_order:
+        mask = df["severity"] == sev
+        if not mask.any():
+            continue
+        sub = df[mask]
+        jitter = rng.uniform(-jitter_scale, jitter_scale, len(sub))
+        x = np.array([x_pos[int(v)] for v in sub["y_true"]]) + jitter
+        ax.scatter(
+            x,
+            sub["prob_fused_c1"].values,
+            c=SEV_COLORS[sev],
+            s=sev_size[sev],
+            alpha=sev_alpha[sev],
+            linewidths=0,
+            zorder=3,
+            label=SEV_LABELS[sev],
+        )
+
+    # Median lines per class
+    for cls, xc in x_pos.items():
+        med = np.median(df.loc[df["y_true"] == cls, "prob_fused_c1"])
+        ax.plot(
+            [xc - jitter_scale - 0.04, xc + jitter_scale + 0.04],
+            [med, med],
+            color="#222",
+            lw=2.0,
+            zorder=5,
+        )
+
+    ax.axhline(0.5, color="#888", lw=1.2, ls="--", alpha=0.7, zorder=2)
+    ax.set_xticks([0, 1])
+    ax.set_xticklabels(["Normal", "Glaucoma"], fontsize=11)
+    ax.set_ylabel("Predicted P(Glaucoma)", fontsize=11)
+    ax.set_ylim(-0.04, 1.04)
+    ax.set_xlim(-0.55, 1.55)
+    ax.set_title(
+        "Confidence Strips — Fused Head\n(Phase 5, all folds)",
+        fontsize=12,
+        fontweight="bold",
+    )
+    ax.grid(axis="y", alpha=0.3, zorder=1)
+
+    # Legend — only show groups that appear
+    handles, labels = ax.get_legend_handles_labels()
+    if handles:
+        ax.legend(
+            handles=handles,
+            labels=labels,
+            fontsize=8.5,
+            loc="upper center",
+            framealpha=0.75,
+            ncol=2,
+        )
+
+    if not has_vfi:
+        ax.text(
+            0.98,
+            0.02,
+            "Re-run with updated v3_hypertower.py\nto enable VFI severity coloring",
+            transform=ax.transAxes,
+            ha="right",
+            va="bottom",
+            fontsize=7.5,
+            color="#888",
+            style="italic",
+        )
+
+    fig.tight_layout()
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(out_path, dpi=180, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {out_path}")
+
+
+# ── Figure 2: Head comparison ─────────────────────────────────────────────────
+
+
+def make_head_comparison(df: pd.DataFrame, out_path: Path) -> None:
+    """Side-by-side violin + strip of P(glaucoma) by true class for each head."""
+    heads = ["fused", "img", "md"]
+    prob_cols = {"fused": "prob_fused_c1", "img": "prob_img_c1", "md": "prob_md_c1"}
+
+    rng = np.random.default_rng(42)
+    fig, axes = plt.subplots(1, 3, figsize=(11, 4.5), sharey=True)
+    fig.patch.set_facecolor("#e8e8e8")
+    fig.suptitle(
+        "Head Comparison — P(Glaucoma) by True Class (Phase 5, all folds)",
+        fontsize=12,
+        fontweight="bold",
+    )
+
+    c_normal = "#4c72b0"
+    c_glaucoma = "#c44e52"
+
+    for ax, head in zip(axes, heads):
+        ax.set_facecolor("#e8e8e8")
+        col = prob_cols[head]
+        data_by_class = [df.loc[df["y_true"] == cls, col].values for cls in [0, 1]]
+
+        vp = ax.violinplot(
+            data_by_class,
+            positions=[0, 1],
+            widths=0.6,
+            showmedians=True,
+            showextrema=False,
+        )
+        for body, color in zip(vp["bodies"], [c_normal, c_glaucoma]):
+            body.set_facecolor(color)
+            body.set_alpha(0.35)
+        vp["cmedians"].set_color("#222")
+        vp["cmedians"].set_linewidth(2)
+
+        for cls, color in zip([0, 1], [c_normal, c_glaucoma]):
+            vals = data_by_class[cls]
+            jitter = rng.uniform(-0.12, 0.12, len(vals))
+            ax.scatter(
+                cls + jitter, vals, color=color, s=4, alpha=0.30, linewidths=0, zorder=3
+            )
+
+        ax.axhline(0.5, color="#888", lw=1.0, ls="--", alpha=0.6)
+        ax.set_xticks([0, 1])
+        ax.set_xticklabels(["Normal", "Glaucoma"], fontsize=9)
+        ax.set_title(
+            HEAD_LABELS[head], fontsize=10, fontweight="bold", color=HEAD_COLORS[head]
+        )
+        ax.set_ylim(-0.05, 1.05)
+        ax.grid(axis="y", alpha=0.3)
+        if head == "fused":
+            ax.set_ylabel("Predicted P(Glaucoma)", fontsize=10)
+
+        from sklearn.metrics import roc_auc_score
+
+        try:
+            auc = roc_auc_score(df["y_true"], df[col])
+            ax.text(
+                0.97,
+                0.04,
+                f"AUC = {auc:.3f}",
+                transform=ax.transAxes,
+                ha="right",
+                va="bottom",
+                fontsize=9,
+                color="#333",
+                bbox=dict(facecolor="white", alpha=0.65, edgecolor="none", pad=2),
+            )
+        except Exception:
+            pass
+
+    fig.tight_layout()
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(out_path, dpi=180, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {out_path}")
+
+
+# ── Multi-model comparison strip ──────────────────────────────────────────────
+
+
+def make_comparison_strips(
+    run_configs: list[dict], vfi: pd.DataFrame, out_path: Path
+) -> None:
+    """
+    Side-by-side confidence strips for multiple runs.
+    Each config: {"label": str, "df": DataFrame}.
+    """
+    from sklearn.metrics import roc_auc_score
+
+    rng = np.random.default_rng(42)
+
+    n = len(run_configs)
+    fig, axes = plt.subplots(1, n, figsize=(4.5 * n, 7), sharey=True)
+    if n == 1:
+        axes = [axes]
+    fig.patch.set_facecolor("#e8e8e8")
+    fig.suptitle(
+        "Confidence Strips by Model (Phase 5, all folds)",
+        fontsize=13,
+        fontweight="bold",
+    )
+
+    sev_order = ["normal", "unknown", "early", "moderate", "severe"]
+    sev_alpha = {
+        "normal": 0.40,
+        "early": 0.55,
+        "moderate": 0.70,
+        "severe": 0.85,
+        "unknown": 0.35,
+    }
+    sev_size = {"normal": 6, "early": 8, "moderate": 10, "severe": 12, "unknown": 6}
+    x_pos = {0: 0.0, 1: 1.0}
+    jitter_scale = 0.18
+
+    for ax, cfg in zip(axes, run_configs):
+        ax.set_facecolor("#e8e8e8")
+        df = cfg["df"]
+
+        # Attach VFI severity
+        df = df.copy()
+        df["patient_id"] = pd.to_numeric(df["patient_id"], errors="coerce").astype(
+            "Int64"
+        )
+        vfi_m = vfi.copy()
+        vfi_m["patient_id"] = vfi_m["patient_id"].astype("Int64")
+        df = df.merge(vfi_m[["patient_id", "severity"]], on="patient_id", how="left")
+        df["severity"] = df["severity"].fillna("unknown")
+
+        for sev in sev_order:
+            mask = df["severity"] == sev
+            if not mask.any():
+                continue
+            sub = df[mask]
+            jitter = rng.uniform(-jitter_scale, jitter_scale, len(sub))
+            x = np.array([x_pos[int(v)] for v in sub["y_true"]]) + jitter
+            ax.scatter(
+                x,
+                sub["prob_fused_c1"].values,
+                c=SEV_COLORS[sev],
+                s=sev_size[sev],
+                alpha=sev_alpha[sev],
+                linewidths=0,
+                zorder=3,
+            )
+
+        # IQR box + median line per class
+        iqr_w = 0.06
+        xticklabels = []
+        for cls, xc in x_pos.items():
+            vals = df.loc[df["y_true"] == cls, "prob_fused_c1"]
+            med = np.median(vals)
+            q25 = np.percentile(vals, 25)
+            q75 = np.percentile(vals, 75)
+            # Subtle translucent IQR box
+            ax.add_patch(
+                plt.Rectangle(
+                    (xc - iqr_w, q25),
+                    2 * iqr_w,
+                    q75 - q25,
+                    facecolor="#555",
+                    alpha=0.18,
+                    linewidth=0,
+                    zorder=4,
+                )
+            )
+            # Median line
+            ax.plot(
+                [xc - jitter_scale - 0.04, xc + jitter_scale + 0.04],
+                [med, med],
+                color="#222",
+                lw=2.0,
+                zorder=5,
+                label="Median" if cls == 0 else None,
+            )
+            # TP / TN rate below x-label
+            if cls == 1:
+                rate = (vals > 0.5).mean() * 100
+                xticklabels.append(f"Glaucoma\nTP {rate:.0f}%")
+            else:
+                rate = (vals <= 0.5).mean() * 100
+                xticklabels.append(f"Normal\nTN {rate:.0f}%")
+
+        ax.axhline(0.5, color="#888", lw=1.2, ls="--", alpha=0.7, zorder=2)
+        ax.set_xticks([0, 1])
+        ax.set_xticklabels(xticklabels, fontsize=10)
+        ax.set_title(cfg["label"], fontsize=11, fontweight="bold")
+        ax.set_ylim(-0.04, 1.04)
+        ax.set_xlim(-0.55, 1.55)
+        ax.grid(axis="y", alpha=0.3, zorder=1)
+
+        try:
+            fold_aucs = [
+                roc_auc_score(g["y_true"], g["prob_fused_c1"])
+                for _, g in df.groupby(["rep", "fold"])
+                if g["y_true"].nunique() > 1
+            ]
+            mean_auc = np.mean(fold_aucs)
+            std_auc = np.std(fold_aucs)
+            ax.text(
+                0.65,
+                0.0,
+                f"AUC = {mean_auc:.3f} ± {std_auc:.3f}",
+                transform=ax.transAxes,
+                ha="right",
+                va="bottom",
+                fontsize=9,
+                color="#333",
+                bbox=dict(facecolor="white", alpha=0.65, edgecolor="none", pad=2),
+            )
+        except Exception:
+            pass
+
+    axes[0].set_ylabel("Predicted P(Glaucoma)", fontsize=11)
+
+    # Shared legend
+    legend_patches = [
+        mpatches.Patch(color=SEV_COLORS[s], label=SEV_LABELS[s])
+        for s in ["normal", "early", "moderate", "severe", "unknown"]
+    ]
+    fig.legend(
+        handles=legend_patches,
+        fontsize=9,
+        loc="lower center",
+        ncol=len(legend_patches),
+        framealpha=0.75,
+        bbox_to_anchor=(0.5, -0.02),
+    )
+
+    fig.tight_layout(rect=[0, 0.06, 1, 1])
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(out_path, dpi=180, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {out_path}")
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+# Default runs shown in the comparison
+DEFAULT_RUNS = [
+    {
+        "run": "phase4/single",
+        "tower_path": "binary/single",
+        "label": "Single HyperTower",
+    },
+    {
+        "run": "phase5/ensemble_fused",
+        "tower_path": "binary/ensemble",
+        "label": "Bilateral Ensemble",
+    },
+    {
+        "run": "phase5/logit_mlp_head",
+        "tower_path": "binary/ensemble",
+        "label": "Fused Head",
+    },
+]
+
+
+def _aggregate_eye_to_patient(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Single-mode predictions are eye-level (2 rows per patient per fold).
+    In the test loader, OD rows come first (sorted patient ID order) then OS.
+    Average the two eyes to get one patient-level row per fold.
+    """
+    rows = []
+    prob_cols = [c for c in df.columns if c.startswith("prob_")]
+    pred_cols = [c for c in df.columns if c.startswith("pred_")]
+
+    for (rep, fold), grp in df.groupby(["rep", "fold"]):
+        n = len(grp)
+        half = n // 2
+        od = grp.iloc[:half].reset_index(drop=True)
+        os_ = grp.iloc[half:].reset_index(drop=True)
+        pat = od.copy()
+        for col in prob_cols:
+            pat[col] = (od[col].values + os_[col].values) / 2
+        for col in pred_cols:
+            pat[col] = (pat[col.replace("pred_", "prob_") + "_c1"] >= 0.5).astype(int)
+        rows.append(pat)
+
+    return pd.concat(rows, ignore_index=True)
+
+
+def _load_run(run: str, tower_path: str, clinical_dir: Path) -> pd.DataFrame:
+    run_dir = RESULTS_ROOT / run
+    print(f"  Loading {run} ...")
+    df = load_all_predictions(run_dir, tower_path=tower_path)
+    if tower_path.endswith("/single"):
+        from v3.scripts.output_analysis.explainability.fold_patient_ids import (
+            attach_patient_ids_single,
+        )
+
+        df = attach_patient_ids_single(df, clinical_dir=clinical_dir, batch_size=8)
+    else:
+        from v3.scripts.output_analysis.explainability.fold_patient_ids import (
+            attach_patient_ids,
+        )
+
+        df = attach_patient_ids(df, clinical_dir=clinical_dir)
+    return df
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    ap.add_argument(
+        "--run", default="phase5/logit_mlp_head", help="Single run for standalone plots"
+    )
+    ap.add_argument("--tower-path", default="binary/ensemble")
+    ap.add_argument("--clinical-dir", type=Path, default=CLINICAL_DIR)
+    ap.add_argument("--out", type=Path, default=FIGURES_ROOT / "explainability")
+    args = ap.parse_args()
+
+    print("Loading VFI data ...")
+    vfi = load_vfi(args.clinical_dir)
+
+    # ── Single-run plots (strips + head comparison) ──────────────────────────
+    df = _load_run(args.run, args.tower_path, args.clinical_dir)
+    make_confidence_strips(df, vfi, args.out / "confidence_strips.png")
+    make_head_comparison(df, args.out / "head_comparison.png")
+
+    # ── Multi-model comparison ───────────────────────────────────────────────
+    print("Building comparison strips ...")
+    run_configs = []
+    for cfg in DEFAULT_RUNS:
+        try:
+            df_r = _load_run(cfg["run"], cfg["tower_path"], args.clinical_dir)
+            run_configs.append({"label": cfg["label"], "df": df_r})
+        except FileNotFoundError as e:
+            print(f"  Skipping {cfg['run']}: {e}")
+    if run_configs:
+        make_comparison_strips(
+            run_configs, vfi, args.out / "confidence_strips_comparison.png"
+        )
+
+
+if __name__ == "__main__":
+    main()

v3/scripts/output_analysis/explainability/fold_patient_ids.py

@@ -0,0 +1,181 @@
+"""
+Derives test-set patient IDs for any (rep, fold) without re-running training.
+
+The split is fully deterministic: fold_seed = rep_seed_start + rep * rep_seed_step.
+build_samples() groups by patient_id with sort=True (pandas default), and the
+test DataLoader uses shuffle=False — so rows in predictions_test.csv are always
+in ascending Patient ID order within each test fold.
+
+Usage:
+    from v3.scripts.output_analysis.explainability.fold_patient_ids import get_test_patient_ids
+    pids = get_test_patient_ids(rep=0, fold=2)   # list of int patient IDs, sorted
+
+    # Attach to a pooled predictions DataFrame:
+    df = attach_patient_ids(df, clinical_dir=...)
+"""
+from __future__ import annotations
+
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+REPO_ROOT    = Path(__file__).resolve().parents[4]
+CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
+
+# These match the defaults in run_cv.py
+_REP_SEED_START = 100
+_REP_SEED_STEP  = 100
+_N_SPLITS       = 5
+_EVAL_MODE      = "binary"
+_LABEL_COL      = "Diagnosis"
+_PATIENT_COL    = "Patient ID"
+
+
+@lru_cache(maxsize=4)
+def _load_clinical(clinical_dir: Path) -> pd.DataFrame:
+    """
+    Load OD/OS Excel sheets, extract Patient ID + Diagnosis, binary-filter.
+    Returns a DataFrame with one row per eye (OD+OS stacked), columns:
+    [Patient ID, Diagnosis, eyeID, VF_MD].
+    """
+    od = pd.read_excel(clinical_dir / "patient_data_od.xlsx", header=1)
+    os_ = pd.read_excel(clinical_dir / "patient_data_os.xlsx", header=1)
+    od["eyeID"] = "OD"
+    os_["eyeID"] = "OS"
+    df = pd.concat([od, os_], ignore_index=True)
+    # Raw column is "ID" (e.g. "#002"); canonicalize to "Patient ID"
+    if "Patient ID" not in df.columns and "ID" in df.columns:
+        df.rename(columns={"ID": "Patient ID"}, inplace=True)
+    df["Patient ID"] = df["Patient ID"].astype(str).str.extract(r"(\d+)")[0].astype(int)
+    df["Diagnosis"]  = pd.to_numeric(df["Diagnosis"], errors="coerce")
+    df["VF_MD"]      = pd.to_numeric(df["VF_MD"],     errors="coerce")
+    # PAPILA encoding: 0=Normal, 1=Glaucoma, 2=Suspect
+    # Binary mode keeps 0 and 1, excludes Suspect (2)
+    df = df[df["Diagnosis"].isin([0, 1])].copy()
+    return df.reset_index(drop=True)
+
+
+def _build_splits(clinical_dir: Path, fold_seed: int) -> list[Any]:
+    """Return list of PatientSplit for a given fold seed."""
+    import sys
+    sys.path.insert(0, str(REPO_ROOT))
+    from v3.classes.split_manager import PatientFirstSplitManager, build_patient_split_plans
+
+    df = _load_clinical(clinical_dir)
+
+    # Patient-level label table (mode label per patient)
+    patient_table = (
+        df.groupby(_PATIENT_COL)[_LABEL_COL]
+        .agg(lambda x: x.mode().iloc[0])
+        .reset_index()
+    )
+    plans_raw = build_patient_split_plans(
+        patient_ids=patient_table[_PATIENT_COL].to_numpy(),
+        patient_labels=patient_table[_LABEL_COL].to_numpy(),
+        n_splits=_N_SPLITS,
+        seed=fold_seed,
+    )
+
+    # Wrap into PatientSplit-like objects with .test DataFrame
+    class _Split:
+        def __init__(self, test_ids):
+            self.test = df[df[_PATIENT_COL].isin(test_ids)].reset_index(drop=True)
+
+    return [_Split(p.test_patient_ids) for p in plans_raw]
+
+
+def get_test_patient_ids(rep: int, fold: int,
+                         clinical_dir: Path = CLINICAL_DIR,
+                         rep_seed_start: int = _REP_SEED_START,
+                         rep_seed_step:  int = _REP_SEED_STEP) -> list[int]:
+    """
+    Return sorted list of Patient IDs in the test set for (rep, fold).
+    Matches the row order of predictions_test.csv for that fold.
+    """
+    fold_seed = rep_seed_start + rep * rep_seed_step
+    plans = _build_splits(clinical_dir, fold_seed)
+    test_df = plans[fold].test
+    # groupby sorts by default → same order as build_samples / test loader
+    return sorted(test_df[_PATIENT_COL].unique().tolist())
+
+
+def _row_to_patient_pos(row_idx: int, n_patients: int, batch_size: int) -> int:
+    """
+    Map a single-mode row index to its patient position in the sorted patient list.
+
+    collect_probs_single_components (aggregate_patient=False) emits predictions
+    in batch-interleaved order: for each batch of B patients, OD rows come first
+    then OS rows.  The last batch may be smaller than batch_size.
+
+        Batch i (B patients): rows [i*2B .. i*2B+B-1] = OD
+                                    [i*2B+B .. i*2B+2B-1] = OS
+        Patient position = i*B + (row_in_batch % B)
+    """
+    full = n_patients // batch_size
+    last_b = n_patients % batch_size
+    for bi in range(full):
+        s = bi * 2 * batch_size
+        if s <= row_idx < s + 2 * batch_size:
+            return bi * batch_size + (row_idx - s) % batch_size
+    if last_b > 0:
+        s = full * 2 * batch_size
+        return full * batch_size + (row_idx - s) % last_b
+    raise IndexError(f"row_idx {row_idx} out of range for n_patients={n_patients}")
+
+
+def attach_patient_ids(df: pd.DataFrame,
+                       clinical_dir: Path = CLINICAL_DIR,
+                       rep_seed_start: int = _REP_SEED_START,
+                       rep_seed_step:  int = _REP_SEED_STEP) -> pd.DataFrame:
+    """
+    Add a 'patient_id' column to a pooled predictions DataFrame.
+    Requires 'rep' and 'fold' columns (added by load_all_predictions).
+    The 'idx' column is the row index within each fold's test set.
+    For patient-level modes (ensemble): idx == patient position directly.
+    """
+    df = df.copy()
+    pid_col = []
+
+    for _, row in df.iterrows():
+        rep_idx  = int(row["rep"].replace("rep", ""))
+        fold_idx = int(row["fold"].replace("fold", ""))
+        idx      = int(row["idx"])
+        pids = get_test_patient_ids(rep_idx, fold_idx,
+                                    clinical_dir=clinical_dir,
+                                    rep_seed_start=rep_seed_start,
+                                    rep_seed_step=rep_seed_step)
+        pid_col.append(pids[idx] if idx < len(pids) else None)
+
+    df["patient_id"] = pid_col
+    return df
+
+
+def attach_patient_ids_single(df: pd.DataFrame,
+                               clinical_dir: Path = CLINICAL_DIR,
+                               batch_size: int = 8,
+                               rep_seed_start: int = _REP_SEED_START,
+                               rep_seed_step:  int = _REP_SEED_STEP) -> pd.DataFrame:
+    """
+    Like attach_patient_ids but for single (eye-level) mode.
+    Single mode emits predictions in batch-interleaved order (see _row_to_patient_pos).
+    batch_size must match the --batch-size used during training (default 8).
+    """
+    df = df.copy()
+    pid_col = []
+
+    for _, row in df.iterrows():
+        rep_idx  = int(row["rep"].replace("rep", ""))
+        fold_idx = int(row["fold"].replace("fold", ""))
+        idx      = int(row["idx"])
+        pids = get_test_patient_ids(rep_idx, fold_idx,
+                                    clinical_dir=clinical_dir,
+                                    rep_seed_start=rep_seed_start,
+                                    rep_seed_step=rep_seed_step)
+        patient_pos = _row_to_patient_pos(idx, len(pids), batch_size)
+        pid_col.append(pids[patient_pos])
+
+    df["patient_id"] = pid_col
+    return df

v3/scripts/output_analysis/explainability/gradcam_phase5.py

@@ -0,0 +1,569 @@
+"""
+GradCAM analysis for Phase 5 — logit_mlp_head checkpointed run.
+
+Produces (all in figures/explainability/gradcam/):
+  mean_cam_normal.png       — average heatmap across all normal test eyes
+  mean_cam_glaucoma.png     — average heatmap across all glaucoma test eyes
+  mean_cam_comparison.png   — side-by-side normal vs glaucoma mean CAMs
+  overlay_grid_normal.png   — grid of individual overlays (normal eyes)
+  overlay_grid_glaucoma.png — grid of individual overlays (glaucoma eyes)
+
+Checkpoints loaded from:
+  v3/results/phase5/logit_mlp_head_ckpt/rep00/binary/ensemble/fold{0..4}/best_single.pt
+
+Usage:
+    python -m v3.scripts.output_analysis.explainability.gradcam_phase5
+"""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.cm as cm
+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+import torch.nn.functional as F
+from PIL import Image
+from tqdm import tqdm
+
+REPO_ROOT     = Path(__file__).resolve().parents[4]
+CKPT_RUN      = REPO_ROOT / "v3" / "results" / "phase5" / "logit_mlp_head_ckpt"
+FIGURES_ROOT  = REPO_ROOT / "v3" / "figures" / "explainability" / "gradcam"
+CLINICAL_DIR  = REPO_ROOT / "Papila" / "ClinicalData"
+IMAGE_DIR     = REPO_ROOT / "Papila" / "FundusImages"
+CONTOUR_DIR   = REPO_ROOT / "Papila" / "ExpertsSegmentations" / "Contours"
+
+DISC_SPAN    = 5    # patch side = DISC_SPAN × disc diameter
+PATCH_SIZE   = 96   # output thumbnail pixels
+
+# Model hyperparameters (inferred from checkpoint weight shapes)
+BACKBONE     = "resnet50"
+NUM_CLASSES  = 2
+CD_HIDDEN    = 128
+FUSION_DIM   = 256
+
+LABEL_NAMES  = {0: "Normal", 1: "Glaucoma"}
+
+
+# ── GradCAM ──────────────────────────────────────────────────────────────────
+
+class GradCAM:
+    """Minimal GradCAM via forward/backward hooks."""
+
+    def __init__(self, target_layer: torch.nn.Module) -> None:
+        self._acts = None
+        self._grads = None
+        self._h1 = target_layer.register_forward_hook(self._save_acts)
+        self._h2 = target_layer.register_full_backward_hook(self._save_grads)
+
+    def _save_acts(self, _m, _i, output):
+        self._acts = output.detach()
+
+    def _save_grads(self, _m, _gi, grad_output):
+        self._grads = grad_output[0].detach()
+
+    def compute(self, img: torch.Tensor, meta: torch.Tensor,
+                model: torch.nn.Module, target_class: int | None = None) -> tuple[np.ndarray, int]:
+        """Return (cam [H,W] normalised 0-1, predicted_class)."""
+        model.eval()
+        with torch.enable_grad():
+            out = model(img, meta)
+            pred = int(out.argmax(1).item())
+            tc = pred if target_class is None else target_class
+            model.zero_grad()
+            out[0, tc].backward()
+
+        weights = self._grads.mean(dim=(2, 3), keepdim=True)
+        cam = F.relu((weights * self._acts).sum(dim=1, keepdim=True))
+        cam = F.interpolate(cam, img.shape[-2:], mode="bilinear", align_corners=False)
+        cam_np = cam.squeeze().cpu().numpy()
+        lo, hi = cam_np.min(), cam_np.max()
+        return (cam_np - lo) / (hi - lo + 1e-8), pred
+
+    def remove(self) -> None:
+        self._h1.remove(); self._h2.remove()
+
+
+def overlay_gradcam(pil: Image.Image, cam: np.ndarray, alpha: float = 0.45) -> Image.Image:
+    cam_u8 = (cam * 255).astype(np.uint8)
+    cam_r = np.array(Image.fromarray(cam_u8).resize(pil.size, Image.BILINEAR)) / 255.0
+    colored = (cm.jet(cam_r)[:, :, :3] * 255).astype(np.uint8)
+    return Image.blend(pil.convert("RGB"), Image.fromarray(colored), alpha)
+
+
+# ── Disc-centred attention helpers ────────────────────────────────────────────
+
+def _disc_contour_path(pid: int, eye: str, expert: int = 1) -> Path:
+    return CONTOUR_DIR / f"RET{pid:03d}{eye}_disc_exp{expert}.txt"
+
+
+def _load_disc_mask(pid: int, eye: str, cam_h: int, cam_w: int) -> np.ndarray | None:
+    """Load expert disc contour, polygon-fill, resize to (cam_h, cam_w)."""
+    from PIL import ImageDraw as _ID
+    p = _disc_contour_path(pid, eye)
+    if not p.exists():
+        return None
+    try:
+        arr = np.loadtxt(str(p), dtype=np.float32)
+    except Exception:
+        return None
+    if arr.ndim == 1:
+        arr = arr.reshape(-1, 2)
+    if arr.shape[0] < 3:
+        return None
+    # Get original image size
+    img_path = get_image_path(pid, eye)
+    try:
+        with Image.open(img_path) as im:
+            orig_w, orig_h = im.size
+    except Exception:
+        return None
+    canvas = Image.new("L", (orig_w, orig_h), 0)
+    _ID.Draw(canvas).polygon([tuple(pt) for pt in arr[:, :2]], fill=1)
+    return np.array(canvas.resize((cam_w, cam_h), Image.NEAREST), dtype=bool)
+
+
+def _disc_centred_patch(cam: np.ndarray, disc_mask: np.ndarray,
+                        span: int = DISC_SPAN, out: int = PATCH_SIZE
+                        ) -> tuple[np.ndarray | None, float | None]:
+    """Translate+scale cam so disc centroid is centred; return (patch, disc_r_out)."""
+    if disc_mask is None or disc_mask.sum() == 0:
+        return None, None
+    ys, xs = np.where(disc_mask)
+    cy, cx = ys.mean(), xs.mean()
+    disc_r = float(np.sqrt(disc_mask.sum() / np.pi))
+    half   = max(1, int(round(span * disc_r / 2)))
+    h, w   = cam.shape
+    y0, y1 = int(round(cy)) - half, int(round(cy)) + half
+    x0, x1 = int(round(cx)) - half, int(round(cx)) + half
+    pt = max(0, -y0); pb = max(0, y1 - h)
+    pl = max(0, -x0); pr = max(0, x1 - w)
+    cam_pad = np.pad(cam, ((pt, pb), (pl, pr)), constant_values=0.0)
+    patch   = cam_pad[y0 + pt: y1 + pt, x0 + pl: x1 + pl]
+    patch_out = np.array(
+        Image.fromarray((np.clip(patch, 0, 1) * 255).astype(np.uint8))
+             .resize((out, out), Image.BILINEAR)
+    ) / 255.0
+    disc_r_out = out * disc_r / (2 * half)
+    return patch_out.astype(np.float32), disc_r_out
+
+
+def make_disc_attention_detail(
+    mean_patches: dict,
+    stats_rows: list[dict],
+    out_path: Path,
+) -> None:  # noqa: C901
+    """
+    2-row (Normal / Glaucoma) × 3-col (correct cam | incorrect cam | disc_frac strip).
+
+    mean_patches: {(cls_name, split): (mean_patch_array, mean_disc_r, count)}
+    stats_rows:   list of {true_name, correct, disc_frac} dicts (floats only, no arrays)
+    """
+    import pandas as pd
+    from matplotlib.patches import Circle
+
+    classes     = ["Normal", "Glaucoma"]
+    splits      = ["correct", "incorrect"]
+    corr_colors = {"correct": "steelblue", "incorrect": "tomato"}
+    stats       = pd.DataFrame(stats_rows)
+
+    # 2 rows (Normal / Glaucoma) × 3 cols (correct cam | incorrect cam | disc_frac strip)
+    fig, axes = plt.subplots(2, 3, figsize=(13, 8),
+                             gridspec_kw={"width_ratios": [1, 1, 0.75]})
+    fig.patch.set_facecolor("#f4f4f4")
+
+    rng = np.random.default_rng(42)
+
+    for ri, cls in enumerate(classes):
+        # Col 0 & 1: correct / incorrect mean CAMs
+        for ci, split in enumerate(splits):
+            ax  = axes[ri, ci]
+            ax.set_facecolor("#222")
+            key = (cls, split)
+            if key in mean_patches:
+                mp, disc_r_out, count = mean_patches[key]
+                ax.imshow(mp, cmap="jet", vmin=0, vmax=1, origin="upper",
+                          extent=[0, PATCH_SIZE, PATCH_SIZE, 0])
+                cx = cy = PATCH_SIZE / 2
+                ax.add_patch(Circle((cx, cy), disc_r_out,
+                                    fill=False, edgecolor="white",
+                                    linewidth=2, linestyle="--"))
+                ax.set_title(f"{split.capitalize()}  (N={count})", fontsize=9)
+            else:
+                ax.text(0.5, 0.5, "no data", ha="center", va="center",
+                        transform=ax.transAxes, fontsize=9, color="grey")
+                ax.set_title(split.capitalize(), fontsize=9)
+            ax.axis("off")
+
+        # Row label on leftmost column
+        axes[ri, 0].set_ylabel(cls, fontsize=11, fontweight="bold", labelpad=8)
+
+        # Col 2: disc_frac strip
+        ax = axes[ri, 2]
+        ax.set_facecolor("#f4f4f4")
+        sub = stats[stats["true_name"] == cls].dropna(subset=["disc_frac"])
+        for xi, split in enumerate(splits):
+            pts = sub[sub["correct"] == (split == "correct")]["disc_frac"].values
+            if len(pts) == 0:
+                continue
+            color  = corr_colors[split]
+            jitter = rng.uniform(-0.18, 0.18, size=len(pts))
+            ax.scatter(xi + jitter, pts, color=color, alpha=0.7, s=28, edgecolors="none")
+            ax.hlines(pts.mean(), xi - 0.28, xi + 0.28, colors=color, linewidth=2.5, zorder=5)
+        ax.set_xticks([0, 1])
+        ax.set_xticklabels(["Correct", "Incorrect"], fontsize=9)
+        ax.set_xlim(-0.55, 1.55)
+        ax.set_ylim(0, 1)
+        ax.set_title("Disc fraction", fontsize=9)
+        ax.grid(axis="y", linestyle="--", alpha=0.3)
+        if ri == 0:
+            ax.set_ylabel("Attention mass inside GT disc", fontsize=9)
+
+    fig.suptitle(
+        "Disc-centred GradCAM attention  |  dashed circle = GT disc boundary\n"
+        "Phase 5, logit_mlp_head, fold 0–4",
+        fontsize=11, fontweight="bold",
+    )
+    fig.tight_layout()
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(out_path, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"Saved: {out_path}")
+
+
+# ── Model loading ─────────────────────────────────────────────────────────────
+
+def build_model(ckpt_path: Path, device: torch.device):
+    """Reconstruct SingleEyeHT from checkpoint and load weights."""
+    from types import SimpleNamespace
+    from v3.classes.models import SingleEyeHT
+    sd = torch.load(ckpt_path, map_location="cpu")
+    # ClinicalTower only reads clinical_data.feature_dim at init time
+    cd_in = sd["cd_tower.block0.0.weight"].shape[1]
+    clinical_shim = SimpleNamespace(feature_dim=cd_in)
+    model = SingleEyeHT(
+        backbone=BACKBONE,
+        freeze_ratio=0.0,
+        augment=False,
+        clinical_data=clinical_shim,
+        num_classes=NUM_CLASSES,
+        cd_hidden_dim=CD_HIDDEN,
+        fusion_dim=FUSION_DIM,
+    )
+    model.load_state_dict(sd)
+    model.to(device).eval()
+    return model
+
+
+# ── Data helpers ──────────────────────────────────────────────────────────────
+
+def build_data_bundle():
+    """Build the PAPILA DataBundle matching the checkpointed run's feature config."""
+    from v3.classes.papila_builders import build_papila_data
+    import torch as _t
+    # Auto-detect cd_in from the majority of checkpoints (excludes stale reps).
+    import collections as _col
+    all_ckpts = list(CKPT_RUN.glob("rep*/binary/ensemble/fold*/best_single.pt"))
+    if all_ckpts:
+        counts = _col.Counter(
+            _t.load(c, map_location="cpu")["cd_tower.block0.0.weight"].shape[1]
+            for c in all_ckpts
+        )
+        cd_in = counts.most_common(1)[0][0]
+    else:
+        cd_in = 25
+    drop_raw = cd_in <= 21
+    excl = ["Axial_Length"] if cd_in in (21, 23) else []
+    return build_papila_data(
+        image_dir=str(IMAGE_DIR),
+        clinical_dir=str(CLINICAL_DIR),
+        label_col="Diagnosis",
+        cat_cols=["Gender", "Phakic/Pseudophakic"],
+        iop_corr_method="ratio",
+        iop_drop_raw=drop_raw,
+        exclude_cols=excl,
+    )
+
+
+def get_image_path(pid: int, eye: str) -> Path:
+    return IMAGE_DIR / f"RET{pid:03d}{eye}.jpg"
+
+
+def build_meta_vector(row, data) -> torch.Tensor:
+    """Build the training-compatible feature vector via DataBundle.vectorize_row."""
+    vec = data.vectorize_row(row)
+    return torch.tensor(vec, dtype=torch.float32).unsqueeze(0)
+
+
+# ── Eval transform ────────────────────────────────────────────────────────────
+
+def get_eval_transform():
+    from torchvision import transforms
+    return transforms.Compose([
+        transforms.Resize(256),
+        transforms.CenterCrop(224),
+        transforms.ToTensor(),
+        transforms.Normalize(mean=(0.485, 0.456, 0.406),
+                             std=(0.229, 0.224, 0.225)),
+    ])
+
+
+# ── Main loop ─────────────────────────────────────────────────────────────────
+
+def _discover_checkpoints(ckpt_run: Path) -> list[tuple[int, int, Path]]:
+    """
+    Scan ckpt_run for all available best_single.pt files.
+    Skips checkpoints whose cd_in doesn't match the majority (to exclude stale reps).
+    Returns sorted list of (rep_idx, fold_idx, ckpt_path).
+    """
+    import collections
+    candidates = []
+    for rep_dir in sorted(ckpt_run.glob("rep*")):
+        try:
+            rep_idx = int(rep_dir.name.replace("rep", ""))
+        except ValueError:
+            continue
+        for fold_dir in sorted((rep_dir / "binary" / "ensemble").glob("fold[0-9]")):
+            ckpt = fold_dir / "best_single.pt"
+            if ckpt.exists():
+                fold_idx = int(fold_dir.name.replace("fold", ""))
+                cd_in = torch.load(ckpt, map_location="cpu")[
+                    "cd_tower.block0.0.weight"
+                ].shape[1]
+                candidates.append((rep_idx, fold_idx, ckpt, cd_in))
+
+    if not candidates:
+        return []
+
+    # Use the majority cd_in so stale reps are automatically excluded
+    counts = collections.Counter(c[3] for c in candidates)
+    target_cd_in = counts.most_common(1)[0][0]
+    skipped = sum(1 for c in candidates if c[3] != target_cd_in)
+    if skipped:
+        print(f"  [discover] skipping {skipped} checkpoint(s) with cd_in≠{target_cd_in}")
+
+    return [(rep, fold, ckpt) for rep, fold, ckpt, cd in candidates if cd == target_cd_in]
+
+
+def run(n_grid: int = 16, alpha: float = 0.45, target_class: int | None = None):
+    """
+    Loop over all available checkpoints in the run directory (all reps × folds).
+    Aggregate CAMs per class, collect overlay grids.
+    """
+    import pandas as pd
+    from v3.scripts.output_analysis.explainability.fold_patient_ids import (
+        get_test_patient_ids,
+    )
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    print(f"Device: {device}")
+
+    print("Building DataBundle ...")
+    data = build_data_bundle()
+    clinical = data.df
+    print(f"  feature_dim={data.feature_dim}  rows={len(clinical)}")
+    transform = get_eval_transform()
+
+    checkpoints = _discover_checkpoints(CKPT_RUN)
+    print(f"Found {len(checkpoints)} checkpoint(s) across "
+          f"{len(set(r for r,f,_ in checkpoints))} rep(s)")
+
+    if not checkpoints:
+        print("No checkpoints found — run with --save-checkpoints first.")
+        return
+
+    # ── Incremental accumulators (no full-res arrays kept after each eye) ────────
+    # Mean CAM per class: running sum
+    cam_sum   = {0: None, 1: None}
+    cam_count = {0: 0,    1: 0}
+
+    # Overlay grid: keep at most n_grid PIL images per class (capped)
+    overlay_items = {0: [], 1: []}
+
+    # Disc-attention detail: running sum of disc patches (not list of arrays)
+    disc_patch_sum   = {}   # (cls_name, split) → np.ndarray sum
+    disc_patch_count = {}   # (cls_name, split) → int
+    disc_radius_sum  = {}   # (cls_name, split) → float sum
+    disc_stats_rows  = []   # floats only — no arrays
+
+    ckpt_bar = tqdm(checkpoints, desc="Folds", unit="fold")
+    for rep_idx, fold_idx, ckpt_path in ckpt_bar:
+        ckpt_bar.set_postfix(rep=rep_idx, fold=fold_idx)
+        model = build_model(ckpt_path, device)
+
+        # GradCAM target: last ResNet block
+        target_layer = model.img_tower.backbone.layer4[-1]
+        gcam = GradCAM(target_layer)
+
+        pids = get_test_patient_ids(rep_idx, fold_idx, clinical_dir=CLINICAL_DIR)
+
+        for pid in tqdm(pids, desc=f"  rep{rep_idx:02d}/fold{fold_idx}", leave=False, unit="pt"):
+            for eye in ("OD", "OS"):
+                img_path = get_image_path(pid, eye)
+                if not img_path.exists():
+                    continue
+
+                row = clinical[
+                    (clinical["Patient ID"] == pid) & (clinical["eyeID"] == eye)
+                ]
+                if len(row) == 0:
+                    continue
+                row = row.iloc[0]
+                label = int(row["Diagnosis"])
+
+                pil_orig = Image.open(img_path).convert("RGB")
+                img_t = transform(pil_orig).unsqueeze(0).to(device)
+                meta_t = build_meta_vector(row, data).to(device)
+
+                cam_np, pred = gcam.compute(img_t, meta_t, model,
+                                            target_class=target_class)
+
+                # Running mean CAM
+                if cam_sum[label] is None:
+                    cam_sum[label] = cam_np.copy()
+                else:
+                    cam_sum[label] += cam_np
+                cam_count[label] += 1
+
+                # Overlay grid — only keep up to n_grid per class
+                if len(overlay_items[label]) < n_grid:
+                    ov = overlay_gradcam(pil_orig, cam_np, alpha=alpha)
+                    overlay_items[label].append((ov, pid, eye, pred))
+
+                # Disc-attention: extract patch now, accumulate into running sum
+                h, w   = cam_np.shape
+                disc_mask = _load_disc_mask(pid, eye, h, w)
+                disc_frac = None
+                if disc_mask is not None and disc_mask.sum() > 0:
+                    disc_frac = float(cam_np[disc_mask].sum() / (cam_np.sum() + 1e-8))
+
+                cls_name = LABEL_NAMES[label]
+                split    = "correct" if (pred == label) else "incorrect"
+                key      = (cls_name, split)
+
+                patch, disc_r_out = _disc_centred_patch(cam_np, disc_mask)
+                if patch is not None:
+                    if key not in disc_patch_sum:
+                        disc_patch_sum[key]   = patch.copy()
+                        disc_patch_count[key] = 1
+                        disc_radius_sum[key]  = disc_r_out
+                    else:
+                        disc_patch_sum[key]   += patch
+                        disc_patch_count[key] += 1
+                        disc_radius_sum[key]  += disc_r_out
+
+                disc_stats_rows.append({
+                    "true_name": cls_name,
+                    "correct":   pred == label,
+                    "disc_frac": disc_frac,
+                })
+
+                # Release per-eye tensors immediately
+                del img_t, meta_t, cam_np
+                if disc_mask is not None:
+                    del disc_mask
+
+        gcam.remove()
+        del model
+        torch.cuda.empty_cache() if torch.cuda.is_available() else None
+
+    # Build mean_patches dict for disc detail plot
+    mean_patches = {
+        key: (
+            disc_patch_sum[key] / disc_patch_count[key],
+            disc_radius_sum[key] / disc_patch_count[key],
+            disc_patch_count[key],
+        )
+        for key in disc_patch_sum
+    }
+
+    # ── Save outputs ─────────────────────────────────────────────────────────
+    FIGURES_ROOT.mkdir(parents=True, exist_ok=True)
+
+    for cls in [0, 1]:
+        if cam_count[cls] == 0:
+            continue
+        mean_cam = cam_sum[cls] / cam_count[cls]
+        lo, hi = mean_cam.min(), mean_cam.max()
+        mean_cam = (mean_cam - lo) / (hi - lo + 1e-8)
+
+        fig, ax = plt.subplots(figsize=(5, 5))
+        ax.imshow(mean_cam, cmap="jet", vmin=0, vmax=1)
+        ax.axis("off")
+        ax.set_title(f"Mean GradCAM — {LABEL_NAMES[cls]}\n(n={cam_count[cls]} eyes, fold 0–4)",
+                     fontsize=11, fontweight="bold")
+        plt.colorbar(ax.images[0], ax=ax, fraction=0.046, pad=0.04)
+        out = FIGURES_ROOT / f"mean_cam_{LABEL_NAMES[cls].lower()}.png"
+        fig.savefig(out, dpi=180, bbox_inches="tight")
+        plt.close(fig)
+        print(f"Saved: {out}")
+
+    # Side-by-side comparison
+    if cam_count[0] > 0 and cam_count[1] > 0:
+        fig, axes = plt.subplots(1, 2, figsize=(10, 5))
+        fig.suptitle("Mean GradCAM — Normal vs Glaucoma (Phase 5, fold 0–4)",
+                     fontsize=12, fontweight="bold")
+        for ax, cls in zip(axes, [0, 1]):
+            mean_cam = cam_sum[cls] / cam_count[cls]
+            lo, hi = mean_cam.min(), mean_cam.max()
+            mean_cam = (mean_cam - lo) / (hi - lo + 1e-8)
+            im = ax.imshow(mean_cam, cmap="jet", vmin=0, vmax=1)
+            ax.axis("off")
+            ax.set_title(f"{LABEL_NAMES[cls]}  (n={cam_count[cls]})", fontsize=11)
+            plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
+        out = FIGURES_ROOT / "mean_cam_comparison.png"
+        fig.savefig(out, dpi=180, bbox_inches="tight")
+        plt.close(fig)
+        print(f"Saved: {out}")
+
+    # Overlay grids
+    for cls in [0, 1]:
+        items = overlay_items[cls]
+        if not items:
+            continue
+        # Sort: misclassified first (more interesting)
+        items.sort(key=lambda x: x[3] == cls)  # wrong preds first
+        items = items[:n_grid]
+        ncols = 4
+        nrows = int(np.ceil(len(items) / ncols))
+        fig, axes = plt.subplots(nrows, ncols, figsize=(ncols * 3.2, nrows * 3.2))
+        axes = np.array(axes).reshape(-1)
+        fig.suptitle(f"GradCAM Overlays — {LABEL_NAMES[cls]} (Phase 5)",
+                     fontsize=12, fontweight="bold")
+        for i, ax in enumerate(axes):
+            if i < len(items):
+                ov, pid, eye, pred = items[i]
+                ax.imshow(ov)
+                correct = pred == cls
+                col = "#2e7d32" if correct else "#c62828"
+                ax.set_title(f"RET{pid:03d}{eye}\n→ {LABEL_NAMES[pred]}",
+                             fontsize=7.5, color=col)
+            ax.axis("off")
+        out = FIGURES_ROOT / f"overlay_grid_{LABEL_NAMES[cls].lower()}.png"
+        fig.tight_layout()
+        fig.savefig(out, dpi=150, bbox_inches="tight")
+        plt.close(fig)
+        print(f"Saved: {out}")
+
+
+    # Disc-centred detail plot
+    if disc_patch_sum:
+        make_disc_attention_detail(mean_patches, disc_stats_rows,
+                                   FIGURES_ROOT / "disc_attention_detail.png")
+
+
+if __name__ == "__main__":
+    import argparse
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--n-grid",       type=int,   default=16,
+                    help="Max overlays per class in grid (default 16)")
+    ap.add_argument("--alpha",        type=float, default=0.45,
+                    help="GradCAM overlay opacity (default 0.45)")
+    ap.add_argument("--target-class", type=int,   default=None,
+                    help="GradCAM target class (default: predicted class)")
+    args = ap.parse_args()
+    run(n_grid=args.n_grid, alpha=args.alpha, target_class=args.target_class)

v3/scripts/output_analysis/explainability/permutation_importance_phase5.py

@@ -0,0 +1,309 @@
+"""
+MD permutation feature importance for Phase 5 — logit_mlp_head checkpointed run.
+
+For each of the 5 fold checkpoints:
+  - loads test images + clinical metadata
+  - caches image features (no grad)
+  - permutes each clinical feature N times and measures AUC drop
+
+Produces (in figures/explainability/):
+  md_importance_phase5.png   — aggregated bar chart across 5 folds
+  md_importance_phase5.csv   — mean/std per feature
+
+Usage:
+    python -m v3.scripts.output_analysis.explainability.permutation_importance_phase5
+"""
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+from types import SimpleNamespace
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+import torch.nn.functional as F
+from sklearn.metrics import roc_auc_score
+
+REPO_ROOT    = Path(__file__).resolve().parents[4]
+CKPT_RUN     = REPO_ROOT / "v3" / "results" / "phase5" / "logit_mlp_head_ckpt"
+FIGURES_ROOT = REPO_ROOT / "v3" / "figures" / "explainability"
+CLINICAL_DIR = REPO_ROOT / "Papila" / "ClinicalData"
+IMAGE_DIR    = REPO_ROOT / "Papila" / "FundusImages"
+
+BACKBONE   = "resnet50"
+NUM_CLASSES = 2
+CD_HIDDEN  = 128
+FUSION_DIM = 256
+N_PERMUTATIONS = 30
+SEED = 0
+
+
+# ── Model ─────────────────────────────────────────────────────────────────────
+
+def build_model(ckpt_path: Path, device: torch.device):
+    from v3.classes.models import SingleEyeHT
+    sd = torch.load(ckpt_path, map_location="cpu")
+    cd_in = sd["cd_tower.block0.0.weight"].shape[1]
+    model = SingleEyeHT(
+        backbone=BACKBONE, freeze_ratio=0.0, augment=False,
+        clinical_data=SimpleNamespace(feature_dim=cd_in),
+        num_classes=NUM_CLASSES, cd_hidden_dim=CD_HIDDEN, fusion_dim=FUSION_DIM,
+    )
+    model.load_state_dict(sd)
+    model.to(device).eval()
+    return model
+
+
+# ── Data ──────────────────────────────────────────────────────────────────────
+
+def build_data_bundle():
+    from v3.classes.papila_builders import build_papila_data
+    # Infer settings from available checkpoint to stay compatible.
+    # Once the 10x5 run (--iop-drop-raw --exclude-cols Axial_Length) completes,
+    # these will automatically match (feature_dim will drop from 25 → 21).
+    ckpt = next(CKPT_RUN.glob("rep*/binary/ensemble/fold*/best_single.pt"), None)
+    import torch as _t
+    cd_in = _t.load(ckpt, map_location="cpu")["cd_tower.block0.0.weight"].shape[1] if ckpt else 25
+    # cd_in=25 → old run (no iop_drop_raw, no excl); cd_in=21 → new run
+    drop_raw = cd_in <= 21
+    excl = ["Axial_Length"] if cd_in in (21, 23) else []
+    return build_papila_data(
+        image_dir=str(IMAGE_DIR), clinical_dir=str(CLINICAL_DIR),
+        label_col="Diagnosis", cat_cols=["Gender", "Phakic/Pseudophakic"],
+        iop_corr_method="ratio", iop_drop_raw=drop_raw, exclude_cols=excl,
+    )
+
+
+def get_eval_transform():
+    from torchvision import transforms
+    return transforms.Compose([
+        transforms.Resize(256), transforms.CenterCrop(224),
+        transforms.ToTensor(),
+        transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
+    ])
+
+
+def get_image_path(pid: int, eye: str) -> Path:
+    return IMAGE_DIR / f"RET{pid:03d}{eye}.jpg"
+
+
+# ── Feature index map ─────────────────────────────────────────────────────────
+
+def build_feature_index_map(data) -> dict[str, list[int]]:
+    """
+    Map feature name → list of dimension indices in the vectorize_row output.
+    Layout: [scalars (min-max scaled)] + [cat one-hots] + [scalar missing flags]
+    """
+    n_scalar = len(data.scalar_cols)
+    cat_expanded = sum(len(m) for m in data.cat_maps.values())
+    feat_map: dict[str, list[int]] = {}
+
+    # Scalar: value dim + missing flag dim
+    for i, col in enumerate(data.scalar_cols):
+        feat_map[col] = [i, n_scalar + cat_expanded + i]
+
+    # Categorical: whole one-hot block
+    cat_offset = n_scalar
+    for col in data.cat_cols:
+        n = len(data.cat_maps[col])
+        feat_map[col] = list(range(cat_offset, cat_offset + n))
+        cat_offset += n
+
+    return feat_map
+
+
+# ── Per-fold importance ───────────────────────────────────────────────────────
+
+def run_fold(rep_idx: int, fold_idx: int, model, data, device: torch.device,
+             n_permutations: int, seed: int) -> dict[str, tuple[float, float]]:
+    """
+    Returns {feature_name: (mean_auc_drop, std_auc_drop)}.
+    """
+    from v3.scripts.output_analysis.explainability.fold_patient_ids import (
+        get_test_patient_ids,
+    )
+    transform = get_eval_transform()
+    clinical = data.df
+
+    pids = get_test_patient_ids(rep_idx, fold_idx, clinical_dir=CLINICAL_DIR)
+
+    # Cache image features + build meta tensors + labels
+    img_feats_list, meta_list, label_list = [], [], []
+    model.eval()
+    with torch.no_grad():
+        for pid in pids:
+            for eye in ("OD", "OS"):
+                img_path = get_image_path(pid, eye)
+                if not img_path.exists():
+                    continue
+                row = clinical[
+                    (clinical["Patient ID"] == pid) & (clinical["eyeID"] == eye)
+                ]
+                if len(row) == 0:
+                    continue
+                row = row.iloc[0]
+                label = int(row["Diagnosis"])
+
+                from PIL import Image
+                pil = Image.open(img_path).convert("RGB")
+                img_t = transform(pil).unsqueeze(0).to(device)
+                feats = model.img_tower(img_t)          # [1, img_dim]
+                meta_vec = torch.tensor(data.vectorize_row(row),
+                                        dtype=torch.float32).unsqueeze(0)
+
+                img_feats_list.append(feats.cpu())
+                meta_list.append(meta_vec)
+                label_list.append(label)
+
+    if not label_list or len(set(label_list)) < 2:
+        print(f"  fold{fold_idx}: insufficient data, skipping.")
+        return {}
+
+    img_feats = torch.cat(img_feats_list).to(device)   # [N, img_dim]
+    meta_all  = torch.cat(meta_list)                    # [N, feat_dim]  on CPU
+    y_true    = np.array(label_list)
+
+    # Baseline AUC
+    with torch.no_grad():
+        md_feats = model.cd_tower(meta_all.to(device))
+        out_f, _, _ = model.bridge(img_feats, md_feats)
+        probs_base = F.softmax(out_f, dim=1)[:, 1].cpu().numpy()
+    baseline_auc = roc_auc_score(y_true, probs_base)
+    print(f"  fold{fold_idx}: baseline AUC={baseline_auc:.4f}  N={len(y_true)}")
+
+    feat_map = build_feature_index_map(data)
+    rng = np.random.default_rng(seed + fold_idx)
+    results: dict[str, tuple[float, float]] = {}
+
+    for feat_name, dims in feat_map.items():
+        drops = []
+        for _ in range(n_permutations):
+            meta_perm = meta_all.clone()
+            perm_idx = rng.permutation(len(meta_perm))
+            meta_perm[:, dims] = meta_perm[perm_idx][:, dims]
+            with torch.no_grad():
+                md_p = model.cd_tower(meta_perm.to(device))
+                out_p, _, _ = model.bridge(img_feats, md_p)
+                probs_p = F.softmax(out_p, dim=1)[:, 1].cpu().numpy()
+            try:
+                drops.append(baseline_auc - roc_auc_score(y_true, probs_p))
+            except Exception:
+                pass
+        if drops:
+            results[feat_name] = (float(np.mean(drops)), float(np.std(drops)))
+
+    return results
+
+
+# ── Aggregate and plot ────────────────────────────────────────────────────────
+
+def plot_importance(all_results: list[dict], out_png: Path, out_csv: Path) -> None:
+    # Aggregate across folds
+    all_feats = sorted({f for r in all_results for f in r})
+    agg = {}
+    for feat in all_feats:
+        vals = [r[feat][0] for r in all_results if feat in r]
+        if vals:
+            agg[feat] = (float(np.mean(vals)), float(np.std(vals)))
+
+    # Sort by mean importance descending
+    sorted_feats = sorted(agg, key=lambda f: agg[f][0], reverse=True)
+    names = sorted_feats
+    imps  = [agg[f][0] for f in names]
+    stds  = [agg[f][1] for f in names]
+    colors = ["#e05c5c" if v >= 0 else "#5c9ee0" for v in imps]
+
+    fig, ax = plt.subplots(figsize=(9, max(4, len(names) * 0.45 + 1.5)))
+    y_pos = np.arange(len(names))
+    ax.barh(y_pos, imps, xerr=stds, color=colors, ecolor="grey", capsize=3, height=0.6)
+    ax.set_yticks(y_pos)
+    ax.set_yticklabels(names, fontsize=9)
+    ax.invert_yaxis()
+    ax.axvline(0, color="black", linewidth=0.8)
+    ax.set_xlabel("Mean AUC drop  (baseline − permuted)", fontsize=10)
+    n_reps  = len(set(r for r in range(len(all_results))))  # placeholder
+    ax.set_title(
+        f"MD Tower — Permutation Feature Importance\n"
+        f"Phase 5 logit_mlp_head_ckpt  ({len(all_results)} folds, "
+        f"error bars = std across folds)",
+        fontsize=11,
+    )
+    fig.tight_layout()
+    out_png.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(out_png, dpi=150, bbox_inches="tight")
+    plt.close(fig)
+    print(f"Saved: {out_png}")
+
+    with open(out_csv, "w", newline="") as f:
+        w = csv.DictWriter(f, fieldnames=["feature", "mean_importance", "std_importance"])
+        w.writeheader()
+        for feat in sorted_feats:
+            w.writerow({"feature": feat,
+                        "mean_importance": agg[feat][0],
+                        "std_importance":  agg[feat][1]})
+    print(f"Saved: {out_csv}")
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def _discover_checkpoints(ckpt_run: Path) -> list[tuple[int, int, Path]]:
+    found = []
+    for rep_dir in sorted(ckpt_run.glob("rep*")):
+        try:
+            rep_idx = int(rep_dir.name.replace("rep", ""))
+        except ValueError:
+            continue
+        for fold_dir in sorted((rep_dir / "binary" / "ensemble").glob("fold[0-9]")):
+            ckpt = fold_dir / "best_single.pt"
+            if ckpt.exists():
+                found.append((rep_idx, int(fold_dir.name.replace("fold", "")), ckpt))
+    return found
+
+
+def main(n_permutations: int = N_PERMUTATIONS, seed: int = SEED):
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    print(f"Device: {device}")
+
+    print("Building DataBundle ...")
+    data = build_data_bundle()
+    print(f"  feature_dim={data.feature_dim}")
+
+    checkpoints = _discover_checkpoints(CKPT_RUN)
+    print(f"Found {len(checkpoints)} checkpoint(s) across "
+          f"{len(set(r for r,f,_ in checkpoints))} rep(s)")
+
+    if not checkpoints:
+        print("No checkpoints found.")
+        return
+
+    all_results = []
+    for rep_idx, fold_idx, ckpt in checkpoints:
+        print(f"\n── rep{rep_idx:02d} fold{fold_idx} ──")
+        model = build_model(ckpt, device)
+        result = run_fold(rep_idx, fold_idx, model, data, device, n_permutations, seed)
+        if result:
+            all_results.append(result)
+        del model
+
+    if not all_results:
+        print("No results — nothing to plot.")
+        return
+
+    plot_importance(
+        all_results,
+        FIGURES_ROOT / "md_importance_phase5.png",
+        FIGURES_ROOT / "md_importance_phase5.csv",
+    )
+
+
+if __name__ == "__main__":
+    import argparse
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--n-permutations", type=int, default=N_PERMUTATIONS)
+    ap.add_argument("--seed",           type=int, default=SEED)
+    args = ap.parse_args()
+    main(n_permutations=args.n_permutations, seed=args.seed)

v3/scripts/output_analysis/plot_architecture.py

@@ -0,0 +1,693 @@
+"""
+Publication-quality architecture diagrams for HyperTower.
+
+Generates:
+  architecture_single_tower.png   — single-eye image-only tower
+  architecture_hypertower.png     — single-eye image + clinical fusion
+  architecture_ensemble.png       — bilateral ensemble (two HyperTowers + average)
+  architecture_fused_head.png     — bilateral ensemble + learned head
+
+Usage:
+    python -m v3.scripts.output_analysis.plot_architecture
+    python -m v3.scripts.output_analysis.plot_architecture --out figures/
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+import matplotlib
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from matplotlib.patches import FancyBboxPatch, FancyArrowPatch
+import matplotlib.patheffects as pe
+
+# ── Colour palette ────────────────────────────────────────────────────────────
+C_IMG    = "#4e8d3a"  # green  — image / CNN
+C_MD     = "#4c72b0"  # blue   — clinical / MLP
+C_BRIDGE = "#c44e52"  # red    — bridge / fusion
+C_EMB    = "#2a9d8f"  # teal   — embedding vectors (z)
+C_OUT    = "#8c6bb1"  # purple — output nodes
+C_HEAD   = "#d4a017"  # gold   — learned head / average
+C_INPUT  = "#a0a0a0"  # grey   — raw input nodes
+C_BG     = "#e8e8e8"
+C_ARROW  = "#444444"
+FONT     = "DejaVu Sans"
+
+
+# ── Low-level primitives ──────────────────────────────────────────────────────
+
+
+def _box(
+    ax,
+    cx,
+    cy,
+    w,
+    h,
+    color,
+    text="",
+    fontsize=9,
+    text_color="white",
+    bold=False,
+    alpha=0.92,
+    radius=0.12,
+    lw=1.5,
+):
+    """Rounded rectangle centered at (cx, cy)."""
+    patch = FancyBboxPatch(
+        (cx - w / 2, cy - h / 2),
+        w,
+        h,
+        boxstyle=f"round,pad=0,rounding_size={radius}",
+        facecolor=color,
+        edgecolor="white",
+        linewidth=lw,
+        alpha=alpha,
+        zorder=3,
+        transform=ax.transData,
+    )
+    ax.add_patch(patch)
+    if text:
+        ax.text(
+            cx,
+            cy,
+            text,
+            ha="center",
+            va="center",
+            fontsize=fontsize,
+            color=text_color,
+            fontweight="bold" if bold else "normal",
+            fontfamily=FONT,
+            zorder=4,
+        )
+    return patch
+
+
+def _arrow(ax, x0, y0, x1, y1, lw=1.6, color=C_ARROW, style="->", rad=0.0):
+    ax.annotate(
+        "",
+        xy=(x1, y1),
+        xytext=(x0, y0),
+        arrowprops=dict(
+            arrowstyle=style,
+            color=color,
+            lw=lw,
+            connectionstyle=f"arc3,rad={rad}",
+        ),
+        zorder=2,
+    )
+
+
+def _text(
+    ax, x, y, s, fontsize=8, color="#333333", ha="center", va="center", bold=False
+):
+    ax.text(
+        x,
+        y,
+        s,
+        ha=ha,
+        va=va,
+        fontsize=fontsize,
+        color=color,
+        fontfamily=FONT,
+        fontweight="bold" if bold else "normal",
+        zorder=5,
+    )
+
+
+def _bracket(
+    ax,
+    x,
+    y0,
+    y1,
+    text="",
+    fontsize=8.5,
+    color="#888888",
+    pad=0.15,
+    lw=1.4,
+    badge_color=None,
+):
+    """Vertical C-bracket on the right side.
+    If badge_color is set, the label is drawn as white text on a filled badge."""
+    mid = (y0 + y1) / 2
+    ax.plot(
+        [x, x + pad, x + pad, x],
+        [y1, y1, y0, y0],
+        color=color,
+        lw=lw,
+        solid_capstyle="round",
+        zorder=2,
+    )
+    if text:
+        if badge_color:
+            ax.text(
+                x + pad * 1.4,
+                mid,
+                text,
+                ha="left",
+                va="center",
+                fontsize=fontsize,
+                color="white",
+                fontfamily=FONT,
+                fontweight="bold",
+                zorder=6,
+                bbox=dict(
+                    facecolor=badge_color,
+                    edgecolor="none",
+                    pad=3.5,
+                    boxstyle="round,pad=0.3",
+                ),
+            )
+        else:
+            ax.text(
+                x + pad * 1.4,
+                mid,
+                text,
+                ha="left",
+                va="center",
+                fontsize=fontsize,
+                color=color,
+                fontfamily=FONT,
+                style="italic",
+            )
+
+
+def _setup(fig, ax, w, h, title):
+    ax.set_xlim(0, w)
+    ax.set_ylim(0, h)
+    ax.axis("off")
+    ax.set_facecolor(C_BG)
+    fig.patch.set_facecolor(C_BG)
+    if title:
+        ax.set_title(
+            title,
+            fontsize=12,
+            fontweight="bold",
+            fontfamily=FONT,
+            pad=10,
+            color="#222222",
+        )
+
+
+# ── Reusable sub-blocks ───────────────────────────────────────────────────────
+
+
+def _draw_cnn_block(ax, x_center, y, w=2.0, h=0.75):
+    """Three-layer CNN block with labels: Conv Layers → Conv Layers → GAP."""
+    labels = ["Conv\nLayers", "Conv\nLayers", "GAP"]
+    sub_w = [w * 0.42, w * 0.30, w * 0.22]
+    sub_h = [h, h * 0.82, h * 0.65]
+    alphas = [0.82, 0.74, 0.66]
+    fsizes = [8.0, 7.5, 7.5]
+    gap = (w - sum(sub_w)) / 2
+    xs = [
+        x_center - w / 2 + sub_w[0] / 2,
+        x_center - w / 2 + sub_w[0] + gap + sub_w[1] / 2,
+        x_center - w / 2 + sub_w[0] + gap + sub_w[1] + gap + sub_w[2] / 2,
+    ]
+    for i, (sx, sw, sh, lbl, alp, fs) in enumerate(
+        zip(xs, sub_w, sub_h, labels, alphas, fsizes)
+    ):
+        _box(ax, sx, y, sw, sh, C_IMG, lbl, fontsize=fs, alpha=alp, radius=0.08)
+        if i < 2:
+            _arrow(
+                ax, sx + sw / 2, y, xs[i + 1] - sub_w[i + 1] / 2, y, lw=1.2, style="-|>"
+            )
+    return xs[-1] + sub_w[-1] / 2
+
+
+def _draw_mlp_block(ax, x_center, y, w=1.4, h=0.65):
+    """Two-layer MLP block: FC(128) → FC(128) (hidden_dim=128 both layers)."""
+    labels = ["FC\n(128)", "FC\n(128)"]
+    w0, w1 = w * 0.55, w * 0.45
+    gap = w - w0 - w1
+    x0 = x_center - w / 2 + w0 / 2
+    x1 = x0 + w0 / 2 + gap + w1 / 2
+    _box(ax, x0, y, w0, h, C_MD, labels[0], fontsize=8.0, alpha=0.82, radius=0.08)
+    _arrow(ax, x0 + w0 / 2, y, x1 - w1 / 2, y, lw=1.2, style="-|>")
+    _box(
+        ax, x1, y, w1, h * 0.88, C_MD, labels[1], fontsize=7.5, alpha=0.72, radius=0.08
+    )
+    return x1 + w1 / 2
+
+
+def _draw_embedding(ax, x, y, w=0.40, h=0.75, label="z\n(emb)"):
+    _box(ax, x + w / 2, y, w, h, C_EMB, label, fontsize=8, bold=True, radius=0.08)
+    return x + w
+
+
+def _draw_output(ax, x, y, dy=0.45, classes=("Glaucoma", "Normal")):
+    """Stacked output class boxes, connected from (x, y) via arrows."""
+    n = len(classes)
+    bw = 1.10
+    bh = 0.38
+    gap = 0.08
+    total = n * bh + (n - 1) * gap
+    y_top = y + total / 2 - bh / 2
+
+    for i, cls in enumerate(classes):
+        cy = y_top - i * (bh + gap)
+        _box(ax, x + bw / 2, cy, bw, bh, C_OUT, cls, fontsize=8.5, radius=0.08)
+        _arrow(ax, x, y, x, cy, lw=1.1, style="-|>", rad=0.0)
+
+    _text(ax, x + bw / 2, y - total / 2 - 0.20, "Softmax", fontsize=7.5, color=C_OUT)
+
+
+def _draw_compact_ht(ax, x_left, y_img, y_md, eye_label):
+    """Compact HyperTower block: Image+Clinical boxes → Bridge.
+    Returns (x_right_of_bridge, y_bridge_center).
+    """
+    bw_img = 1.40
+    bh_img = 0.72
+    bw_md = 1.20
+    bh_md = 0.62
+    bw_br = 0.72
+    cy_br = (y_img + y_md) / 2
+    bh_br = abs(y_img - y_md) * 0.60
+
+    # Image box: CNN Backbone
+    _box(
+        ax,
+        x_left + bw_img / 2,
+        y_img,
+        bw_img,
+        bh_img,
+        C_IMG,
+        f"{eye_label}\nCNN Backbone",
+        fontsize=8.5,
+        radius=0.08,
+    )
+    # MD box: Clinical MLP
+    _box(
+        ax,
+        x_left + bw_md / 2,
+        y_md,
+        bw_md,
+        bh_md,
+        C_MD,
+        f"{eye_label}\nClinical MLP",
+        fontsize=8.5,
+        radius=0.08,
+    )
+
+    # Arrows to bridge
+    br_x = x_left + max(bw_img, bw_md) + 0.60
+    _arrow(ax, x_left + bw_img, y_img, br_x - bw_br / 2, cy_br, lw=1.2, style="-|>")
+    _arrow(ax, x_left + bw_md, y_md, br_x - bw_br / 2, cy_br, lw=1.2, style="-|>")
+
+    # Bridge label kept simple — detail lives in the hypertower diagram
+    _box(
+        ax,
+        br_x,
+        cy_br,
+        bw_br,
+        max(bh_br, 0.70),
+        C_BRIDGE,
+        "Bridge",
+        fontsize=8.0,
+        radius=0.08,
+    )
+
+    return br_x + bw_br / 2, cy_br
+
+
+# ── Figure 1: Single Tower ────────────────────────────────────────────────────
+
+
+def make_single_tower(out_dir: Path):
+    W, H = 9.0, 3.2
+    fig, ax = plt.subplots(figsize=(W, H))
+    _setup(fig, ax, W, H, "Single Tower")
+
+    cy = H / 2
+
+    # Input
+    _box(
+        ax,
+        0.75,
+        cy,
+        0.95,
+        0.60,
+        C_INPUT,
+        "Fundus\nImage",
+        fontsize=8.5,
+        radius=0.08,
+        alpha=0.75,
+        text_color="#333",
+    )
+    _arrow(ax, 1.22, cy, 1.60, cy)
+
+    # CNN Backbone
+    cnn_x_right = _draw_cnn_block(ax, x_center=3.10, y=cy, w=2.80, h=0.78)
+    _text(ax, 3.10, cy - 0.68, "CNN Backbone", fontsize=8.5, color=C_IMG, bold=True)
+    _arrow(ax, 1.60, cy, 1.73, cy, lw=1.4, style="-|>")
+
+    # Embedding
+    emb_x_right = _draw_embedding(ax, x=cnn_x_right + 0.28, y=cy, w=0.48, h=0.78)
+    _arrow(ax, cnn_x_right, cy, cnn_x_right + 0.28, cy, lw=1.4, style="-|>")
+
+    # Classifier
+    _arrow(ax, emb_x_right, cy, emb_x_right + 0.25, cy, lw=1.4, style="-|>")
+    _draw_output(ax, emb_x_right + 0.25, cy)
+
+    path = out_dir / "architecture_single_tower.png"
+    fig.savefig(path, dpi=180, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+# ── Figure 2: HyperTower (single eye) ────────────────────────────────────────
+
+
+def make_hypertower(out_dir: Path):
+    W, H = 11.0, 5.5
+    fig, ax = plt.subplots(figsize=(W, H))
+    _setup(fig, ax, W, H, "HyperTower — Single Eye")
+
+    y_img = 3.70
+    y_md = 1.60
+
+    # ── Image tower ────────────────────────────────────────────────
+    _box(
+        ax,
+        0.80,
+        y_img,
+        1.00,
+        0.60,
+        C_INPUT,
+        "Fundus\nImage",
+        fontsize=8.5,
+        radius=0.08,
+        alpha=0.75,
+        text_color="#333",
+    )
+    _arrow(ax, 1.30, y_img, 1.85, y_img)
+    cnn_x_r = _draw_cnn_block(ax, x_center=3.50, y=y_img, w=2.80, h=0.75)
+    _text(ax, 3.50, y_img - 0.65, "CNN Backbone", fontsize=8, color=C_IMG, bold=True)
+    _arrow(ax, 1.85, y_img, 1.98, y_img, lw=1.4, style="-|>")
+
+    emb_img_x = _draw_embedding(ax, x=cnn_x_r + 0.30, y=y_img, w=0.65, h=0.75)
+    _arrow(ax, cnn_x_r, y_img, cnn_x_r + 0.30, y_img, lw=1.4, style="-|>")
+    _text(
+        ax,
+        (1.30 + emb_img_x) / 2,
+        y_img + 0.65,
+        "Image Tower",
+        fontsize=9,
+        color=C_IMG,
+        bold=True,
+    )
+
+    # ── Clinical tower ─────────────────────────────────────────────
+    _box(
+        ax,
+        0.80,
+        y_md,
+        1.00,
+        0.55,
+        C_INPUT,
+        "Clinical\nData",
+        fontsize=8.5,
+        radius=0.08,
+        alpha=0.75,
+        text_color="#333",
+    )
+    _arrow(ax, 1.30, y_md, 1.65, y_md)
+    mlp_x_r = _draw_mlp_block(ax, x_center=2.90, y=y_md, w=1.60, h=0.65)
+    _arrow(ax, 1.65, y_md, 1.74, y_md, lw=1.4, style="-|>")
+
+    emb_md_x = _draw_embedding(ax, x=mlp_x_r + 0.30, y=y_md, w=0.65, h=0.65)
+    _arrow(ax, mlp_x_r, y_md, mlp_x_r + 0.30, y_md, lw=1.4, style="-|>")
+    _text(
+        ax,
+        (1.30 + emb_md_x) / 2,
+        y_md - 0.60,
+        "Clinical Tower",
+        fontsize=9,
+        color=C_MD,
+        bold=True,
+    )
+
+    # ── Bridge ─────────────────────────────────────────────────────
+    br_x = max(emb_img_x, emb_md_x) + 0.80
+    cy_br = (y_img + y_md) / 2
+    bh_br = abs(y_img - y_md) * 0.55
+
+    _arrow(ax, emb_img_x, y_img, br_x - 0.40, cy_br, lw=1.4, style="-|>")
+    _arrow(ax, emb_md_x, y_md, br_x - 0.40, cy_br, lw=1.4, style="-|>")
+    _box(
+        ax,
+        br_x,
+        cy_br,
+        1.40,
+        max(bh_br, 1.35),
+        C_BRIDGE,
+        "Bridge\nFC(img→256)\nFC(md→256)\n⊙ Hadamard\n→ ReLU→FC(2)",
+        fontsize=8,
+        radius=0.10,
+    )
+
+    # ── Output ─────────────────────────────────────────────────────
+    out_x = br_x + 0.65 + 0.40
+    _arrow(ax, br_x + 0.65, cy_br, out_x, cy_br, lw=1.4, style="-|>")
+    _draw_output(ax, out_x, cy_br)
+
+    # ── Bracket (right of output nodes; output bw=1.10 so right edge = out_x+1.10)
+    _bracket(
+        ax,
+        x=out_x + 1.25,
+        y0=y_md - 0.50,
+        y1=y_img + 0.50,
+        text="HyperTower",
+        fontsize=9,
+        pad=0.22,
+        color="#555",
+        badge_color="#555",
+    )
+
+    path = out_dir / "architecture_hypertower.png"
+    fig.savefig(path, dpi=180, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+# ── Figure 3: Bilateral Ensemble ─────────────────────────────────────────────
+
+
+def make_ensemble(out_dir: Path):
+    W, H = 10.5, 7.5
+    fig, ax = plt.subplots(figsize=(W, H))
+    _setup(fig, ax, W, H, "Bilateral Ensemble HyperTower")
+
+    x_left = 3.0
+    inp_cx = 1.85
+    inp_w  = 0.90
+    inp_h  = 0.55
+
+    # OD (top)
+    od_y_img, od_y_md = 5.90, 4.60
+    br_od_x, cy_od = _draw_compact_ht(
+        ax, x_left=x_left, y_img=od_y_img, y_md=od_y_md, eye_label="OD"
+    )
+    _text(ax, 0.45, (od_y_img + od_y_md) / 2, "OD\n(Right Eye)",
+          fontsize=9, color="#444", bold=True)
+    _box(ax, inp_cx, od_y_img, inp_w, inp_h, C_INPUT, "Fundus\nImage",
+         fontsize=8, radius=0.08, alpha=0.75, text_color="#333")
+    _box(ax, inp_cx, od_y_md, inp_w, inp_h, C_INPUT, "Clinical\nData",
+         fontsize=8, radius=0.08, alpha=0.75, text_color="#333")
+    _arrow(ax, inp_cx + inp_w / 2, od_y_img, x_left, od_y_img, lw=1.2, style="-|>")
+    _arrow(ax, inp_cx + inp_w / 2, od_y_md,  x_left, od_y_md,  lw=1.2, style="-|>")
+
+    # OS (bottom)
+    os_y_img, os_y_md = 2.80, 1.50
+    br_os_x, cy_os = _draw_compact_ht(
+        ax, x_left=x_left, y_img=os_y_img, y_md=os_y_md, eye_label="OS"
+    )
+    _text(ax, 0.45, (os_y_img + os_y_md) / 2, "OS\n(Left Eye)",
+          fontsize=9, color="#444", bold=True)
+    _box(ax, inp_cx, os_y_img, inp_w, inp_h, C_INPUT, "Fundus\nImage",
+         fontsize=8, radius=0.08, alpha=0.75, text_color="#333")
+    _box(ax, inp_cx, os_y_md, inp_w, inp_h, C_INPUT, "Clinical\nData",
+         fontsize=8, radius=0.08, alpha=0.75, text_color="#333")
+    _arrow(ax, inp_cx + inp_w / 2, os_y_img, x_left, os_y_img, lw=1.2, style="-|>")
+    _arrow(ax, inp_cx + inp_w / 2, os_y_md,  x_left, os_y_md,  lw=1.2, style="-|>")
+
+    # Average node
+    avg_x = max(br_od_x, br_os_x) + 1.20
+    avg_y = (cy_od + cy_os) / 2
+    avg_size = 0.90
+
+    _arrow(ax, br_od_x, cy_od, avg_x - avg_size / 2, avg_y, lw=1.4, style="-|>")
+    _arrow(ax, br_os_x, cy_os, avg_x - avg_size / 2, avg_y, lw=1.4, style="-|>")
+    _box(
+        ax,
+        avg_x,
+        avg_y,
+        avg_size,
+        avg_size,
+        C_HEAD,
+        "Average",
+        fontsize=10,
+        bold=True,
+        radius=0.10,
+    )
+
+    # Output
+    out_x = avg_x + avg_size / 2 + 0.50
+    _arrow(ax, avg_x + avg_size / 2, avg_y, out_x, avg_y, lw=1.5, style="-|>")
+    _draw_output(ax, out_x, avg_y)
+
+    # Side brackets — white text on badge
+    _bracket(
+        ax,
+        x=br_od_x + 0.10,
+        y0=od_y_md - 0.45,
+        y1=od_y_img + 0.45,
+        text="OD HyperTower",
+        fontsize=8.5,
+        pad=0.20,
+        color="#555",
+        badge_color="#555",
+    )
+    _bracket(
+        ax,
+        x=br_os_x + 0.10,
+        y0=os_y_md - 0.45,
+        y1=os_y_img + 0.45,
+        text="OS HyperTower",
+        fontsize=8.5,
+        pad=0.20,
+        color="#555",
+        badge_color="#555",
+    )
+
+    path = out_dir / "architecture_ensemble.png"
+    fig.savefig(path, dpi=180, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+# ── Figure 4: Fused Head ──────────────────────────────────────────────────────
+
+
+def make_fused_head(out_dir: Path):
+    W, H = 10.5, 7.5
+    fig, ax = plt.subplots(figsize=(W, H))
+    _setup(fig, ax, W, H, "Fused Head Bilateral HyperTower")
+
+    x_left = 3.0
+    inp_cx = 1.85
+    inp_w  = 0.90
+    inp_h  = 0.55
+
+    # OD (top) — same layout as ensemble
+    od_y_img, od_y_md = 5.90, 4.60
+    br_od_x, cy_od = _draw_compact_ht(
+        ax, x_left=x_left, y_img=od_y_img, y_md=od_y_md, eye_label="OD"
+    )
+    _text(ax, 0.45, (od_y_img + od_y_md) / 2, "OD\n(Right Eye)",
+          fontsize=9, color="#444", bold=True)
+    _box(ax, inp_cx, od_y_img, inp_w, inp_h, C_INPUT, "Fundus\nImage",
+         fontsize=8, radius=0.08, alpha=0.75, text_color="#333")
+    _box(ax, inp_cx, od_y_md, inp_w, inp_h, C_INPUT, "Clinical\nData",
+         fontsize=8, radius=0.08, alpha=0.75, text_color="#333")
+    _arrow(ax, inp_cx + inp_w / 2, od_y_img, x_left, od_y_img, lw=1.2, style="-|>")
+    _arrow(ax, inp_cx + inp_w / 2, od_y_md,  x_left, od_y_md,  lw=1.2, style="-|>")
+
+    # OS (bottom)
+    os_y_img, os_y_md = 2.80, 1.50
+    br_os_x, cy_os = _draw_compact_ht(
+        ax, x_left=x_left, y_img=os_y_img, y_md=os_y_md, eye_label="OS"
+    )
+    _text(ax, 0.45, (os_y_img + os_y_md) / 2, "OS\n(Left Eye)",
+          fontsize=9, color="#444", bold=True)
+    _box(ax, inp_cx, os_y_img, inp_w, inp_h, C_INPUT, "Fundus\nImage",
+         fontsize=8, radius=0.08, alpha=0.75, text_color="#333")
+    _box(ax, inp_cx, os_y_md, inp_w, inp_h, C_INPUT, "Clinical\nData",
+         fontsize=8, radius=0.08, alpha=0.75, text_color="#333")
+    _arrow(ax, inp_cx + inp_w / 2, os_y_img, x_left, os_y_img, lw=1.2, style="-|>")
+    _arrow(ax, inp_cx + inp_w / 2, os_y_md,  x_left, os_y_md,  lw=1.2, style="-|>")
+
+    _bracket(
+        ax,
+        x=br_od_x + -0.17,
+        y0=od_y_md - 0.45,
+        y1=od_y_img + 0.45,
+        text="OD HyperTower",
+        fontsize=8.5,
+        pad=0.20,
+        color="#555",
+        badge_color="#555",
+    )
+    _bracket(
+        ax,
+        x=br_os_x + -0.17,
+        y0=os_y_md - 0.45,
+        y1=os_y_img + 0.45,
+        text="OS HyperTower",
+        fontsize=8.5,
+        pad=0.20,
+        color="#555",
+        badge_color="#555",
+    )
+
+    # Fused Head box with logit MLP detail
+    avg_y = (cy_od + cy_os) / 2
+    head_x = max(br_od_x, br_os_x) + 2.20
+    head_w = 1.80
+    head_h = 1.20
+
+    _arrow(ax, br_od_x + 0.05, cy_od, head_x - head_w / 2, avg_y, lw=1.4, style="-|>")
+    _arrow(ax, br_os_x + 0.05, cy_os, head_x - head_w / 2, avg_y, lw=1.4, style="-|>")
+    _box(
+        ax,
+        head_x,
+        avg_y,
+        head_w,
+        head_h,
+        C_HEAD,
+        "Fused Head\ncat(l_OD, l_OS)\n→ FC(64) → logits",
+        fontsize=8.5,
+        bold=False,
+        radius=0.10,
+    )
+
+    # Output nodes + softmax
+    out_x = head_x + head_w / 2 + 0.50
+    _arrow(ax, head_x + head_w / 2, avg_y, out_x, avg_y, lw=1.5, style="-|>")
+    _draw_output(ax, out_x, avg_y)
+
+    path = out_dir / "architecture_fused_head.png"
+    fig.savefig(path, dpi=180, bbox_inches="tight")
+    plt.close(fig)
+    print(f"  Saved: {path}")
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    ap.add_argument(
+        "--out",
+        type=Path,
+        default=Path(__file__).resolve().parents[3] / "v3" / "figures",
+        help="Output directory",
+    )
+    args = ap.parse_args()
+    args.out.mkdir(parents=True, exist_ok=True)
+
+    print("Generating architecture diagrams...")
+    make_single_tower(args.out)
+    make_hypertower(args.out)
+    make_ensemble(args.out)
+    make_fused_head(args.out)
+    print("Done.")
+
+
+if __name__ == "__main__":
+    main()

v3/scripts/output_analysis/plot_cnn_backbone_comparison.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+"""
+Bar plot comparing CNN standalone vs HyperTower image_only AUC per backbone,
+with PAPILA paper reference lines.
+
+Usage:
+  python -m v3.scripts.output_analysis.plot_cnn_backbone_comparison \
+      --results-dir v3/results/phase1 \
+      --output v3/results/phase1/cnn_backbone_comparison.png
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from sklearn.metrics import roc_auc_score
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
+
+
+BACKBONES = ["densenet121", "vgg16", "mobilenet_v2", "inception_v3", "resnet50"]
+BACKBONE_LABELS = {
+    "densenet121":  "DenseNet121",
+    "vgg16":        "VGG16",
+    "mobilenet_v2": "MobileNetV2",
+    "inception_v3": "Inception V3",
+    "resnet50":     "ResNet50",
+}
+
+# Per-backbone paper AUCs (binary, Test #2, PAPILA 2022)
+PAPER_AUC = {
+    "densenet121":  0.80,
+    "vgg16":        0.84,
+    "mobilenet_v2": 0.75,
+    "inception_v3": 0.78,
+    "resnet50":     0.78,
+}
+PAPER_STD = {
+    "densenet121":  0.05,
+    "vgg16":        0.02,
+    "mobilenet_v2": 0.06,
+    "inception_v3": 0.08,
+    "resnet50":     0.07,
+}
+
+COLOURS = {
+    "cnn":       "#4878CF",
+    "ht":        "#D65F5F",
+    "paper_ref": "black",
+}
+
+
+def load_cnn_fold_aucs(results_dir: Path, backbone: str) -> list[float]:
+    fpath = results_dir / f"cnn_{backbone}" / "fold_metrics.csv"
+    if not fpath.exists():
+        print(f"  WARNING: missing {fpath}")
+        return []
+    df = pd.read_csv(fpath)
+    return df["auc"].tolist()
+
+
+def load_ht_fold_aucs(results_dir: Path, backbone: str, n_folds: int = 5) -> list[float]:
+    aucs = []
+    for fold in range(n_folds):
+        fold_dir = results_dir / "imageonly_ht" / backbone / "binary" / "single" / f"fold{fold}"
+        y_path  = fold_dir / "test_y_true.npy"
+        p_path  = fold_dir / "test_probs_fused.npy"
+        if not (y_path.exists() and p_path.exists()):
+            print(f"  WARNING: missing predictions for {backbone} fold{fold}")
+            continue
+        y  = np.load(y_path)
+        pr = np.load(p_path)
+        if len(np.unique(y)) < 2:
+            print(f"  WARNING: single-class test set for {backbone} fold{fold}, skipping")
+            continue
+        aucs.append(float(roc_auc_score(y, pr[:, 1])))
+    return aucs
+
+
+def plot(cnn_data: dict, ht_data: dict, output: Path):
+    n = len(BACKBONES)
+    x = np.arange(n)
+    group_width = 0.7
+    bar_w = group_width / 2 * 0.88
+    offsets = [-group_width / 4, group_width / 4]
+
+    fig, ax = plt.subplots(figsize=(10, 5.5))
+
+    for bi, backbone in enumerate(BACKBONES):
+        for si, (tag, data, colour) in enumerate([
+            ("CNN standalone", cnn_data, COLOURS["cnn"]),
+            ("HyperTower (image only)", ht_data, COLOURS["ht"]),
+        ]):
+            aucs = data.get(backbone, [])
+            if not aucs:
+                continue
+            xpos = bi + offsets[si]
+            mean, std = np.mean(aucs), np.std(aucs)
+            ax.bar(
+                xpos, mean, width=bar_w,
+                color=colour, alpha=0.80,
+                label=tag if bi == 0 else "_nolegend_",
+            )
+            ax.errorbar(
+                xpos, mean, yerr=std,
+                fmt="none", color="black", capsize=4, linewidth=1.2,
+            )
+
+        # Paper reference line spanning this backbone's group
+        paper_val = PAPER_AUC.get(backbone)
+        if paper_val is not None:
+            lw = group_width / 2 + bar_w / 2
+            label = "PAPILA paper" if bi == 0 else "_nolegend_"
+            ax.hlines(
+                paper_val,
+                bi - group_width / 2, bi + group_width / 2,
+                colors=COLOURS["paper_ref"], linestyles=":", linewidths=1.8,
+                label=label,
+            )
+
+    ax.set_xticks(x)
+    ax.set_xticklabels([BACKBONE_LABELS[b] for b in BACKBONES], fontsize=11)
+    ax.set_ylabel("AUC (ROC)", fontsize=11)
+    ax.set_title("Phase 1: CNN backbone AUC — standalone vs HyperTower (image only)", fontsize=12)
+    ax.set_ylim(0.45, 1.02)
+    ax.axhline(0.5, color="grey", linestyle="--", linewidth=0.8, alpha=0.4)
+    ax.grid(axis="y", alpha=0.3, linestyle="--")
+    ax.legend(loc="lower right", fontsize=10, framealpha=0.9)
+    fig.tight_layout()
+
+    output.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(output, dpi=180)
+    plt.close(fig)
+    print(f"Saved: {output}")
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--results-dir", default="v3/results/phase1")
+    ap.add_argument("--output", default=None)
+    args = ap.parse_args()
+
+    results_dir = Path(args.results_dir)
+    output = Path(args.output) if args.output else results_dir / "cnn_backbone_comparison.png"
+
+    cnn_data = {b: load_cnn_fold_aucs(results_dir, b) for b in BACKBONES}
+    ht_data  = {b: load_ht_fold_aucs(results_dir, b)  for b in BACKBONES}
+
+    plot(cnn_data, ht_data, output)
+
+    # Summary table
+    print(f"\n{'Backbone':<16}  {'CNN standalone':>18}  {'HT image_only':>18}  {'Paper':>12}")
+    print("-" * 72)
+    for b in BACKBONES:
+        cnn_aucs = cnn_data[b]
+        ht_aucs  = ht_data[b]
+        cnn_str  = f"{np.mean(cnn_aucs):.3f} ± {np.std(cnn_aucs):.3f}" if cnn_aucs else "—"
+        ht_str   = f"{np.mean(ht_aucs):.3f} ± {np.std(ht_aucs):.3f}"  if ht_aucs  else "—"
+        p_str    = f"{PAPER_AUC[b]:.2f} ± {PAPER_STD[b]:.2f}"
+        print(f"{BACKBONE_LABELS[b]:<16}  {cnn_str:>18}  {ht_str:>18}  {p_str:>12}")
+
+
+if __name__ == "__main__":
+    main()

v3/scripts/output_analysis/plot_phase3_modality_ablation.py

@@ -0,0 +1,119 @@
+"""
+Phase 3 modality ablation — Image-only vs Clinical-only vs HyperTower (fused).
+
+Pools all rep×fold predictions from the phase3/baseline run and plots
+per-fold AUC for each modality as a box plot with jittered points.
+
+Output: v3/figures/phase3_modality_ablation.png
+
+Usage:
+    python -m v3.scripts.output_analysis.plot_phase3_modality_ablation
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from sklearn.metrics import roc_auc_score
+
+REPO_ROOT    = Path(__file__).resolve().parents[3]
+RESULTS_DIR  = REPO_ROOT / "v3" / "results" / "phase3" / "baseline"
+FIGURES_DIR  = REPO_ROOT / "v3" / "figures"
+OUT_PNG      = FIGURES_DIR / "phase3_modality_ablation.png"
+
+C_BASELINE = "#dd8452"
+C_OTHER    = "#4c72b0"
+C_MEDIAN   = "#c44e52"
+FSIZE      = 10
+
+MODALITIES = [
+    ("prob_img_c1",   "Image only",    C_OTHER),
+    ("prob_md_c1",    "Clinical only", C_OTHER),
+    ("prob_fused_c1", "HyperTower\n(fused)", C_BASELINE),
+]
+
+
+def load_fold_aucs() -> dict[str, list[float]]:
+    aucs: dict[str, list[float]] = {col: [] for col, _, _ in MODALITIES}
+
+    for rep_dir in sorted(RESULTS_DIR.glob("rep*")):
+        fold_root = rep_dir / "binary" / "single"
+        if not fold_root.exists():
+            continue
+        for fold_dir in sorted(fold_root.glob("fold[0-9]")):
+            csv = fold_dir / "predictions_test.csv"
+            if not csv.exists():
+                continue
+            df = pd.read_csv(csv)
+            if df["y_true"].nunique() < 2:
+                continue
+            for col, _, _ in MODALITIES:
+                if col in df.columns:
+                    try:
+                        aucs[col].append(roc_auc_score(df["y_true"], df[col]))
+                    except Exception:
+                        pass
+
+    return aucs
+
+
+def main():
+    print("Loading fold AUCs ...")
+    aucs = load_fold_aucs()
+
+    n_folds = len(next(iter(aucs.values())))
+    print(f"  {n_folds} folds found")
+    for col, label, _ in MODALITIES:
+        vals = aucs[col]
+        print(f"  {label.replace(chr(10), ' '):<30}  "
+              f"mean={np.mean(vals):.4f}  std={np.std(vals):.4f}  n={len(vals)}")
+
+    # ── Plot ──────────────────────────────────────────────────────────────────
+    fig, ax = plt.subplots(figsize=(6, 4.5))
+
+    data_list = [np.array(aucs[col]) for col, _, _ in MODALITIES]
+    colors     = [color for _, _, color in MODALITIES]
+    labels     = [lbl   for _, lbl, _ in MODALITIES]
+    x          = np.arange(len(MODALITIES))
+
+    bp = ax.boxplot(
+        data_list,
+        vert=True,
+        patch_artist=True,
+        positions=x,
+        widths=0.3,
+        showfliers=True,
+        flierprops=dict(marker="o", markersize=3, alpha=0.5),
+        medianprops=dict(color=C_MEDIAN, linewidth=2),
+    )
+    for patch, color in zip(bp["boxes"], colors):
+        patch.set_facecolor(color)
+        patch.set_alpha(0.8)
+
+    ax.set_xlim(-0.5, len(MODALITIES) - 0.5)
+    tick_labels = [
+        f"{lbl}\nAUC={np.mean(np.array(aucs[col])):.3f}"
+        for col, lbl, _ in MODALITIES
+    ]
+    ax.set_xticks(x)
+    ax.set_xticklabels(tick_labels, fontsize=FSIZE)
+    ax.set_ylabel("AUC (ROC)", fontsize=FSIZE + 1)
+    fig.suptitle(
+        f"Phase 3 — Modality Ablation: Image / Clinical / Fused  ({n_folds} folds)",
+        fontsize=FSIZE + 3, fontweight="bold",
+    )
+    ax.grid(axis="y", alpha=0.3)
+
+    fig.tight_layout()
+    FIGURES_DIR.mkdir(parents=True, exist_ok=True)
+    fig.savefig(OUT_PNG, dpi=180, bbox_inches="tight")
+    plt.close(fig)
+    print(f"Saved: {OUT_PNG}")
+
+
+if __name__ == "__main__":
+    main()