pre-refactor 041426

v3/scripts/main/phase2/phase_2_overnight.sh

@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+# Phase 2 overnight batch — image-only ResNet50, 10x5 rep-CV
+# Runs 6 configurations:
+#   1. Leaky CV (eye-level splits)
+#   2. Proper CV (patient-level, baseline)
+#   3. GT crop  scale=1.1  (paper-matched tight crop)
+#   4. GT crop  scale=2.5  (default generous crop)
+#   5. UNet crop scale=1.1
+#   6. UNet crop scale=2.5
+set -euo pipefail
+
+SCRIPT="python -m v3.scripts.main.run_cv"
+OUTROOT="v3/results/phase2"
+MANIFEST="manifest.csv"
+UNET_WEIGHTS="models/v2/refuge/segmentation/per_image/best.pt"
+
+BASE="--eval-mode binary \
+      --tower-mode single \
+      --bridge-mode image_only \
+      --backbone resnet50 \
+      --epochs 30 \
+      --augment \
+      --in-memory-cache \
+      --reps 10 \
+      --rep-seed-start 100 \
+      --rep-seed-step 100 \
+      --output-root ${OUTROOT}"
+
+echo "============================================================"
+echo " Phase 2 overnight batch"
+echo " $(date)"
+echo "============================================================"
+
+# ----------------------------------------------------------------
+# 1. Leaky CV (eye-level splits, no crop)
+# ----------------------------------------------------------------
+echo ""
+echo "=== [1/6] Leaky CV (eye-level) ==="
+$SCRIPT $BASE \
+    --leaky-cv \
+    --run-name imageonly_resnet50_leaky
+
+# ----------------------------------------------------------------
+# 2. Proper CV (patient-level, no crop) — baseline
+# ----------------------------------------------------------------
+echo ""
+echo "=== [2/6] Proper CV (patient-level, baseline) ==="
+$SCRIPT $BASE \
+    --run-name imageonly_resnet50_proper
+
+# ----------------------------------------------------------------
+# 3. GT crop, scale=1.1  (paper-matched tight crop)
+# ----------------------------------------------------------------
+echo ""
+echo "=== [3/6] GT crop, scale=1.1 ==="
+$SCRIPT $BASE \
+    --img-crop-gt \
+    --img-crop-manifest ${MANIFEST} \
+    --img-crop-scale 1.1 \
+    --img-crop-size 200 \
+    --run-name imageonly_resnet50_gtcrop_1.1
+
+# ----------------------------------------------------------------
+# 4. GT crop, scale=2.5  (default generous crop)
+# ----------------------------------------------------------------
+echo ""
+echo "=== [4/6] GT crop, scale=2.5 ==="
+$SCRIPT $BASE \
+    --img-crop-gt \
+    --img-crop-manifest ${MANIFEST} \
+    --img-crop-scale 2.5 \
+    --img-crop-size 200 \
+    --run-name imageonly_resnet50_gtcrop_2.5
+
+# ----------------------------------------------------------------
+# 5. UNet crop, scale=1.1
+# ----------------------------------------------------------------
+echo ""
+echo "=== [5/6] UNet crop, scale=1.1 ==="
+$SCRIPT $BASE \
+    --img-crop-weights ${UNET_WEIGHTS} \
+    --img-crop-manifest ${MANIFEST} \
+    --img-crop-scale 1.1 \
+    --img-crop-size 200 \
+    --run-name imageonly_resnet50_unetcrop_1.1
+
+# ----------------------------------------------------------------
+# 6. UNet crop, scale=2.5
+# ----------------------------------------------------------------
+echo ""
+echo "=== [6/6] UNet crop, scale=2.5 ==="
+$SCRIPT $BASE \
+    --img-crop-weights ${UNET_WEIGHTS} \
+    --img-crop-manifest ${MANIFEST} \
+    --img-crop-scale 2.5 \
+    --img-crop-size 200 \
+    --run-name imageonly_resnet50_unetcrop_2.5
+
+# ----------------------------------------------------------------
+# 7. Refugelike backbone (proper CV, no crop) — pre-training effect
+# ----------------------------------------------------------------
+echo ""
+echo "=== [7/7] Refugelike backbone (proper CV, no crop) ==="
+$SCRIPT $BASE \
+    --backbone refugelike \
+    --run-name imageonly_refugelike_proper
+
+echo ""
+echo "============================================================"
+echo " All done — $(date)"
+echo "============================================================"

v3/scripts/main/phase3/dispatch_phase3.py

@@ -0,0 +1,197 @@
+"""
+Dispatch phase 3 experiment runs to the distributed job server.
+
+Reads experiment_grid.json, checks which runs already have complete 10x5 results,
+and submits the rest via submit-cv. Skips runs marked needs_implementation.
+
+Usage:
+    python -m v3.scripts.main.phase3.dispatch_phase3 \
+        --server http://hades:8765 --token hypertower
+
+    # Dry run (print what would be submitted, don't actually submit):
+    python -m v3.scripts.main.phase3.dispatch_phase3 \
+        --server http://hades:8765 --token hypertower --dry-run
+
+    # Override number of reps (default 10):
+    python -m v3.scripts.main.phase3.dispatch_phase3 \
+        --server http://hades:8765 --token hypertower --reps 4
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+import requests
+
+# Allow running as `python v3/scripts/main/phase3/dispatch_phase3.py`
+sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
+
+GRID_PATH   = Path(__file__).parent / "experiment_grid.json"
+RESULTS_ROOT = Path(__file__).resolve().parents[4] / "v3" / "results"
+MODULE      = "v3.scripts.main.run_cv"
+OUTPUT_DIR  = "v3/results"
+REP_SEED_START = 100
+REP_SEED_STEP  = 100
+
+
+# ── Completion check ──────────────────────────────────────────────────────────
+
+def _completed_reps(run_name: str, reps: int) -> list[int]:
+    """Return list of rep indices that already have a summary.json."""
+    done = []
+    for i in range(reps):
+        summary = RESULTS_ROOT / run_name / f"rep{i:02d}" / "binary" / "single" / "summary.json"
+        if summary.exists():
+            done.append(i)
+    return done
+
+
+# ── Server API ────────────────────────────────────────────────────────────────
+
+class _API:
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self._h = {"x-token": token}
+
+    def get(self, path: str, **params) -> object:
+        r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+    def post(self, path: str, body: dict) -> dict:
+        r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+
+def _queued_reps(jobs: list[dict], run_name: str) -> set[int]:
+    """Return rep indices already pending or running in the server queue."""
+    active = set()
+    for job in jobs:
+        if job["run_name"] != run_name:
+            continue
+        if job["state"] not in ("pending", "running"):
+            continue
+        # Extract --rep-index from job args
+        try:
+            args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"])
+            if "--rep-index" in args:
+                active.add(int(args[args.index("--rep-index") + 1]))
+        except Exception:
+            pass
+    return active
+
+
+def _submit_cv(api: _API, run_name: str, run_args: list[str],
+               reps: int, missing: list[int], dry_run: bool):
+    """Submit one job per missing rep."""
+    for i in missing:
+        seed = REP_SEED_START + i * REP_SEED_STEP
+        rep_args = run_args + [
+            "--run-name", run_name,
+            "--reps", "1",
+            "--rep-seed-start", str(seed),
+            "--rep-index", str(i),
+        ]
+        body = {
+            "run_name": run_name,
+            "module": MODULE,
+            "args": rep_args,
+            "output_dir": OUTPUT_DIR,
+            "priority": 0,
+        }
+        if dry_run:
+            print(f"  [dry-run] would queue rep{i:02d}  seed={seed}")
+        else:
+            resp = api.post("/jobs", body)
+            print(f"  queued rep{i:02d}  seed={seed}  job_id={resp['job_id']}")
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                  formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""),
+                    help="Server URL (or set HT_SERVER)")
+    ap.add_argument("--token",  default=os.environ.get("HT_TOKEN",  ""),
+                    help="Shared secret (or set HT_TOKEN)")
+    ap.add_argument("--reps",   type=int, default=10,
+                    help="Expected number of reps per run (default: 10)")
+    ap.add_argument("--grid",   type=Path, default=GRID_PATH,
+                    help="Path to experiment grid JSON (default: experiment_grid.json)")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="Print what would be submitted without actually submitting")
+    args = ap.parse_args()
+
+    if not args.dry_run:
+        if not args.server:
+            ap.error("--server is required (or set HT_SERVER)")
+        if not args.token:
+            ap.error("--token is required (or set HT_TOKEN)")
+    elif not args.server or not args.token:
+        print("[dry-run] no --server/--token provided — skipping queue check, showing disk state only")
+
+    grid = json.loads(args.grid.read_text())
+    common_args = grid["common_args"]
+    api = _API(args.server, args.token) if (args.server and args.token) else None
+
+    # Fetch current server queue once (pending + running)
+    server_jobs: list[dict] = []
+    if api:
+        try:
+            all_jobs = api.get("/jobs")
+            server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")]
+            print(f"[server] {len(server_jobs)} job(s) currently pending/running in queue")
+        except Exception as e:
+            print(f"[warn] could not fetch server queue: {e}")
+
+    # Collect all runs: baseline + every group's runs
+    all_runs = [grid["baseline"]]
+    for group in grid["groups"]:
+        if group.get("needs_implementation"):
+            print(f"\n[skip] group '{group['name']}' — {group['needs_implementation']}")
+            continue
+        all_runs.extend(group["runs"])
+
+    submitted_total = 0
+    skipped_total   = 0
+
+    for run in all_runs:
+        run_name  = run["run_name"]
+        run_args  = common_args + run.get("extra_args", [])
+        done      = set(_completed_reps(run_name, args.reps))
+        queued    = _queued_reps(server_jobs, run_name)
+        accounted = done | queued
+        missing   = [i for i in range(args.reps) if i not in accounted]
+
+        if not missing:
+            if len(done) == args.reps:
+                print(f"\n[done]  {run_name}  ({args.reps}/{args.reps} reps complete)")
+            else:
+                in_q = sorted(queued - done)
+                print(f"\n[skip]  {run_name}  ({len(done)} done, {len(in_q)} queued: {[f'rep{i:02d}' for i in in_q]})")
+            skipped_total += 1
+            continue
+
+        parts = []
+        if done:    parts.append(f"{len(done)} done")
+        if queued:  parts.append(f"{len(queued - done)} queued")
+        status = ", ".join(parts) if parts else "not started"
+        print(f"\n[queue] {run_name}  ({status}) — submitting {len(missing)} rep(s)")
+        _submit_cv(api, run_name, run_args, args.reps, missing, args.dry_run)
+        submitted_total += len(missing)
+
+    print(f"\n{'='*50}")
+    print(f"Submitted: {submitted_total} jobs  |  Already accounted for: {skipped_total} runs")
+    if grid.get("groups"):
+        needs_impl = sum(1 for g in grid["groups"] if g.get("needs_implementation"))
+        if needs_impl:
+            print(f"Skipped (needs implementation): {needs_impl} group(s)")
+
+
+if __name__ == "__main__":
+    main()

v3/scripts/main/phase3/epoch_grid.json

@@ -0,0 +1,66 @@
+{
+  "_notes": [
+    "Epoch length sensitivity experiments.",
+    "All other settings match the phase3 baseline (fused bridge, BCD p=0.5, refugelike, etc.).",
+    "common_args are prepended to every run's args list."
+  ],
+
+  "common_args": [
+    "--eval-mode", "binary",
+    "--tower-mode", "single",
+    "--in-memory-cache",
+    "--augment",
+    "--tune-binary-threshold",
+    "--backbone", "refugelike",
+    "--output-root", "v3/results"
+  ],
+
+  "_common_args_implicit_defaults": {
+    "--bridge-mode": "fused",
+    "--tower-loss-mode": "bcd",
+    "--bcd-prob": "0.5",
+    "--warmup-cd-epochs": "40",
+    "--single-warmup-tower-epochs": "3",
+    "--single-warmup-fused-epochs": "3"
+  },
+
+  "baseline": {
+    "run_name": "phase3/epochs_30",
+    "description": "30-epoch run — same as phase3 baseline, included here for direct comparison.",
+    "extra_args": ["--epochs", "30"]
+  },
+
+  "groups": [
+    {
+      "name": "epoch_length",
+      "description": "Test sensitivity to total training epochs (warmup epochs unchanged).",
+      "runs": [
+        {
+          "run_name": "phase3/epochs_1",
+          "description": "1 epoch total — effectively pure warmup output with a single main-phase step.",
+          "extra_args": ["--epochs", "1"]
+        },
+        {
+          "run_name": "phase3/epochs_5",
+          "description": "5 epochs total.",
+          "extra_args": ["--epochs", "5"]
+        },
+        {
+          "run_name": "phase3/epochs_10",
+          "description": "10 epochs total.",
+          "extra_args": ["--epochs", "10"]
+        },
+        {
+          "run_name": "phase3/epochs_20",
+          "description": "20 epochs total.",
+          "extra_args": ["--epochs", "20"]
+        },
+        {
+          "run_name": "phase3/epochs_50",
+          "description": "50 epochs total.",
+          "extra_args": ["--epochs", "50"]
+        }
+      ]
+    }
+  ]
+}

v3/scripts/main/phase3/experiment_grid.json

@@ -0,0 +1,362 @@
+{
+  "_notes": [
+    "All runs use single-eye tower mode, binary eval, 10x5 rep-CV.",
+    "common_args are prepended to every run's args list.",
+    "Entries marked 'needs_implementation' require small code changes before running (noted inline).",
+    "Bridge fusion uses elementwise product of projected image/clinical features.",
+    "SE infrastructure exists in Bridge/ImageTower/ClinicalTower but use_se is hardcoded False",
+    "  in SingleEyeHT \u2014 add --se-img-tower / --se-cd-tower / --se-bridge flags to wire through.",
+    "Dropout is hardcoded: Bridge classifier=0.5, ClinicalTower=0.1 \u2014 add --bridge-dropout /",
+    "  --cd-dropout flags to make configurable."
+  ],
+  "common_args": [
+    "--eval-mode",
+    "binary",
+    "--tower-mode",
+    "single",
+    "--epochs",
+    "30",
+    "--in-memory-cache",
+    "--augment",
+    "--tune-binary-threshold",
+    "--backbone",
+    "refugelike",
+    "--output-root",
+    "v3/results"
+  ],
+  "_common_args_implicit_defaults": {
+    "--bridge-mode": "fused",
+    "--tower-loss-mode": "bcd",
+    "--bcd-prob": "0.5",
+    "--warmup-cd-epochs": "40",
+    "--single-warmup-tower-epochs": "3",
+    "--single-warmup-fused-epochs": "3"
+  },
+  "baseline": {
+    "run_name": "phase3/baseline",
+    "description": "Fused bridge (image+clinical), BCD p=0.5, cd_warmup=40, tower/fused warmup=3/3, no SE, no IOP correction.",
+    "extra_args": []
+  },
+  "groups": [
+    {
+      "name": "loss_function",
+      "description": "Test BCD loss variants vs cross-entropy baseline.",
+      "runs": [
+        {
+          "run_name": "phase3/loss_all",
+          "description": "All-losses mode (cross-entropy on all three heads every step).",
+          "extra_args": [
+            "--tower-loss-mode",
+            "all"
+          ]
+        },
+        {
+          "run_name": "phase3/loss_bcd_p03",
+          "description": "BCD with lower switching probability (more CE, less BCD).",
+          "extra_args": [
+            "--tower-loss-mode",
+            "bcd",
+            "--bcd-prob",
+            "0.3"
+          ]
+        },
+        {
+          "run_name": "phase3/loss_bcd_p07",
+          "description": "BCD with higher switching probability (more BCD, less CE).",
+          "extra_args": [
+            "--tower-loss-mode",
+            "bcd",
+            "--bcd-prob",
+            "0.7"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "se_attention",
+      "description": "Squeeze-and-excitation gates at different points in the network.",
+      "runs": [
+        {
+          "run_name": "phase3/se_bridge",
+          "description": "SE gate on fused vector inside the bridge only.",
+          "extra_args": [
+            "--se-bridge"
+          ]
+        },
+        {
+          "run_name": "phase3/se_img_tower",
+          "description": "SE gate on image tower output features.",
+          "extra_args": [
+            "--se-img-tower"
+          ]
+        },
+        {
+          "run_name": "phase3/se_cd_tower",
+          "description": "SE gate on clinical tower output features.",
+          "extra_args": [
+            "--se-cd-tower"
+          ]
+        },
+        {
+          "run_name": "phase3/se_all",
+          "description": "SE gates on image tower, clinical tower, and bridge.",
+          "extra_args": [
+            "--se-img-tower",
+            "--se-cd-tower",
+            "--se-bridge"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "iop_correction",
+      "description": "Test different IOP measurement correction strategies (default: no correction).",
+      "runs": [
+        {
+          "run_name": "phase3/iop_ratio",
+          "description": "IOP correction via Perkins\u2192Pneumatic ratio scaling.",
+          "extra_args": [
+            "--iop-corr-method",
+            "ratio"
+          ]
+        },
+        {
+          "run_name": "phase3/iop_ols",
+          "description": "IOP correction via OLS regression.",
+          "extra_args": [
+            "--iop-corr-method",
+            "ols"
+          ]
+        },
+        {
+          "run_name": "phase3/iop_lad",
+          "description": "IOP correction via LAD (robust to outliers) regression.",
+          "extra_args": [
+            "--iop-corr-method",
+            "lad"
+          ]
+        },
+        {
+          "run_name": "phase3/iop_multi",
+          "description": "IOP correction via multivariate regression including pachymetry.",
+          "extra_args": [
+            "--iop-corr-method",
+            "multi"
+          ]
+        },
+        {
+          "run_name": "phase3/iop_ratio_drop_raw",
+          "description": "Ratio correction + drop raw IOP (only corrected IOP seen by model).",
+          "extra_args": [
+            "--iop-corr-method",
+            "ratio",
+            "--iop-drop-raw"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "feature_ablation",
+      "description": "Exclude individual clinical features to measure each one's contribution.",
+      "runs": [
+        {
+          "run_name": "phase3/excl_iop",
+          "description": "No IOP features \u2014 tests how much intraocular pressure contributes.",
+          "extra_args": [
+            "--exclude-cols",
+            "IOP",
+            "Pachymetry"
+          ]
+        },
+        {
+          "run_name": "phase3/excl_age",
+          "description": "No age feature.",
+          "extra_args": [
+            "--exclude-cols",
+            "Age"
+          ]
+        },
+        {
+          "run_name": "phase3/excl_axial_length",
+          "description": "No axial length feature.",
+          "extra_args": [
+            "--exclude-cols",
+            "Axial_Length"
+          ]
+        },
+        {
+          "run_name": "phase3/excl_refractive",
+          "description": "No refractive defect feature.",
+          "extra_args": [
+            "--exclude-cols",
+            "Refractive_Defect"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "network_dims",
+      "description": "Test sensitivity to clinical tower and bridge fusion dimensionality.",
+      "runs": [
+        {
+          "run_name": "phase3/cd_hidden_64",
+          "description": "Smaller clinical tower (64 hidden units vs default 128).",
+          "extra_args": [
+            "--cd-hidden-dim",
+            "64"
+          ]
+        },
+        {
+          "run_name": "phase3/cd_hidden_256",
+          "description": "Larger clinical tower (256 hidden units vs default 128).",
+          "extra_args": [
+            "--cd-hidden-dim",
+            "256"
+          ]
+        },
+        {
+          "run_name": "phase3/fusion_dim_128",
+          "description": "Smaller fusion space (128 vs default 256).",
+          "extra_args": [
+            "--fusion-dim",
+            "128"
+          ]
+        },
+        {
+          "run_name": "phase3/fusion_dim_512",
+          "description": "Larger fusion space (512 vs default 256).",
+          "extra_args": [
+            "--fusion-dim",
+            "512"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "backbone_freezing",
+      "description": "Partial backbone freezing to reduce overfitting and speed training.",
+      "runs": [
+        {
+          "run_name": "phase3/freeze_25",
+          "description": "Freeze earliest 25% of backbone blocks.",
+          "extra_args": [
+            "--freeze-ratio",
+            "0.25"
+          ]
+        },
+        {
+          "run_name": "phase3/freeze_50",
+          "description": "Freeze earliest 50% of backbone blocks.",
+          "extra_args": [
+            "--freeze-ratio",
+            "0.50"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "learning_rate",
+      "description": "Test LR sensitivity (default 1e-4).",
+      "runs": [
+        {
+          "run_name": "phase3/lr_1e3",
+          "description": "Higher learning rate 1e-3.",
+          "extra_args": [
+            "--lr",
+            "1e-3"
+          ]
+        },
+        {
+          "run_name": "phase3/lr_3e4",
+          "description": "Intermediate learning rate 3e-4.",
+          "extra_args": [
+            "--lr",
+            "3e-4"
+          ]
+        },
+        {
+          "run_name": "phase3/lr_1e5",
+          "description": "Lower learning rate 1e-5.",
+          "extra_args": [
+            "--lr",
+            "1e-5"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "dropout",
+      "description": "Test bridge classifier and clinical tower dropout rates.",
+      "runs": [
+        {
+          "run_name": "phase3/bridge_dropout_03",
+          "description": "Reduce bridge classifier dropout from 0.5 to 0.3.",
+          "extra_args": [
+            "--bridge-dropout",
+            "0.3"
+          ]
+        },
+        {
+          "run_name": "phase3/bridge_dropout_07",
+          "description": "Increase bridge classifier dropout to 0.7.",
+          "extra_args": [
+            "--bridge-dropout",
+            "0.7"
+          ]
+        },
+        {
+          "run_name": "phase3/cd_dropout_03",
+          "description": "Increase clinical tower dropout from 0.1 to 0.3.",
+          "extra_args": [
+            "--cd-dropout",
+            "0.3"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "warmup",
+      "description": "Test warmup ablations vs default (cd=40, tower/fused=3/3).",
+      "runs": [
+        {
+          "run_name": "phase3/warmup_no_cd",
+          "description": "No CD warmup (cd=0) \u2014 tests whether the 40-epoch CD warmup is necessary.",
+          "extra_args": [
+            "--warmup-cd-epochs",
+            "0"
+          ]
+        },
+        {
+          "run_name": "phase3/warmup_tower5_fused5",
+          "description": "Extended tower/fused warmup (5/5 vs default 3/3).",
+          "extra_args": [
+            "--single-warmup-tower-epochs",
+            "5",
+            "--single-warmup-fused-epochs",
+            "5"
+          ]
+        }
+      ]
+    },
+    {
+      "name": "sampling_augmentation",
+      "description": "Test data sampling and augmentation choices.",
+      "runs": [
+        {
+          "run_name": "phase3/no_augment",
+          "description": "No augmentation \u2014 baseline images only.",
+          "extra_args": [
+            "--no-augment"
+          ]
+        },
+        {
+          "run_name": "phase3/balanced_sampling",
+          "description": "Weighted balanced sampler to counter class imbalance.",
+          "extra_args": [
+            "--balanced-sampling"
+          ]
+        }
+      ]
+    }
+  ]
+}

v3/scripts/main/phase3/phase35_grid.json

@@ -0,0 +1,54 @@
+{
+  "_notes": [
+    "Phase 3.5 — confirmation and BCD tuning.",
+    "All runs use best settings from phase 3: refugelike backbone, ratio IOP correction, drop raw IOP, exclude axial length.",
+    "Baseline here is phase3/iop_ratio_drop_raw (0.8685 ± 0.011) — already complete, not re-run.",
+    "common_args are prepended to every run's args list."
+  ],
+
+  "common_args": [
+    "--eval-mode", "binary",
+    "--tower-mode", "single",
+    "--epochs", "30",
+    "--in-memory-cache",
+    "--augment",
+    "--tune-binary-threshold",
+    "--backbone", "refugelike",
+    "--iop-corr-method", "ratio",
+    "--iop-drop-raw",
+    "--exclude-cols", "Axial_Length",
+    "--output-root", "v3/results"
+  ],
+
+  "_common_args_implicit_defaults": {
+    "--bridge-mode": "fused",
+    "--warmup-cd-epochs": "40",
+    "--single-warmup-tower-epochs": "3",
+    "--single-warmup-fused-epochs": "3"
+  },
+
+  "baseline": {
+    "run_name": "phase35/iop_bcd_p07",
+    "description": "Best IOP preprocessing + best BCD prob from phase 3 combined.",
+    "extra_args": ["--tower-loss-mode", "bcd", "--bcd-prob", "0.7"]
+  },
+
+  "groups": [
+    {
+      "name": "bcd_tuning",
+      "description": "Extended BCD probability sweep with best IOP settings.",
+      "runs": [
+        {
+          "run_name": "phase35/iop_bcd_p08",
+          "description": "BCD p=0.8 with ratio IOP + drop raw.",
+          "extra_args": ["--tower-loss-mode", "bcd", "--bcd-prob", "0.8"]
+        },
+        {
+          "run_name": "phase35/iop_bcd_p09",
+          "description": "BCD p=0.9 with ratio IOP + drop raw.",
+          "extra_args": ["--tower-loss-mode", "bcd", "--bcd-prob", "0.9"]
+        }
+      ]
+    }
+  ]
+}

v3/scripts/main/phase4/dispatch_phase4.py

@@ -0,0 +1,199 @@
+"""
+Dispatch phase 4 experiment runs to the distributed job server.
+
+Reads experiment_grid.json, checks which runs already have complete 10x5 results,
+and submits the rest. Skips runs marked needs_implementation.
+
+Usage:
+    python -m v3.scripts.main.phase4.dispatch_phase4 \
+        --server http://hades:8765 --token hypertower
+
+    # Dry run (print what would be submitted, don't actually submit):
+    python -m v3.scripts.main.phase4.dispatch_phase4 \
+        --server http://hades:8765 --token hypertower --dry-run
+
+    # Override number of reps (default 10):
+    python -m v3.scripts.main.phase4.dispatch_phase4 \
+        --server http://hades:8765 --token hypertower --reps 4
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+import requests
+
+# Allow running as `python v3/scripts/main/phase3/dispatch_phase3.py`
+sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
+
+GRID_PATH   = Path(__file__).parent / "experiment_grid.json"
+RESULTS_ROOT = Path(__file__).resolve().parents[4] / "v3" / "results"
+MODULE      = "v3.scripts.main.run_cv"
+OUTPUT_DIR  = "v3/results"
+REP_SEED_START = 100
+REP_SEED_STEP  = 100
+
+
+# ── Completion check ──────────────────────────────────────────────────────────
+
+def _completed_reps(run_name: str, reps: int) -> list[int]:
+    """Return list of rep indices that already have a summary.json (any tower mode)."""
+    done = []
+    for i in range(reps):
+        rep_dir = RESULTS_ROOT / run_name / f"rep{i:02d}" / "binary"
+        # Accept any tower mode subdir
+        if rep_dir.exists() and any((rep_dir / tm / "summary.json").exists()
+                                     for tm in ("single", "bilateral", "siamese", "ensemble")):
+            done.append(i)
+    return done
+
+
+# ── Server API ────────────────────────────────────────────────────────────────
+
+class _API:
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self._h = {"x-token": token}
+
+    def get(self, path: str, **params) -> object:
+        r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+    def post(self, path: str, body: dict) -> dict:
+        r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+
+def _queued_reps(jobs: list[dict], run_name: str) -> set[int]:
+    """Return rep indices already pending or running in the server queue."""
+    active = set()
+    for job in jobs:
+        if job["run_name"] != run_name:
+            continue
+        if job["state"] not in ("pending", "running"):
+            continue
+        # Extract --rep-index from job args
+        try:
+            args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"])
+            if "--rep-index" in args:
+                active.add(int(args[args.index("--rep-index") + 1]))
+        except Exception:
+            pass
+    return active
+
+
+def _submit_cv(api: _API, run_name: str, run_args: list[str],
+               reps: int, missing: list[int], dry_run: bool):
+    """Submit one job per missing rep."""
+    for i in missing:
+        seed = REP_SEED_START + i * REP_SEED_STEP
+        rep_args = run_args + [
+            "--run-name", run_name,
+            "--reps", "1",
+            "--rep-seed-start", str(seed),
+            "--rep-index", str(i),
+        ]
+        body = {
+            "run_name": run_name,
+            "module": MODULE,
+            "args": rep_args,
+            "output_dir": OUTPUT_DIR,
+            "priority": 0,
+        }
+        if dry_run:
+            print(f"  [dry-run] would queue rep{i:02d}  seed={seed}")
+        else:
+            resp = api.post("/jobs", body)
+            print(f"  queued rep{i:02d}  seed={seed}  job_id={resp['job_id']}")
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                  formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""),
+                    help="Server URL (or set HT_SERVER)")
+    ap.add_argument("--token",  default=os.environ.get("HT_TOKEN",  ""),
+                    help="Shared secret (or set HT_TOKEN)")
+    ap.add_argument("--reps",   type=int, default=10,
+                    help="Expected number of reps per run (default: 10)")
+    ap.add_argument("--grid",   type=Path, default=GRID_PATH,
+                    help="Path to experiment grid JSON (default: experiment_grid.json)")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="Print what would be submitted without actually submitting")
+    args = ap.parse_args()
+
+    if not args.dry_run:
+        if not args.server:
+            ap.error("--server is required (or set HT_SERVER)")
+        if not args.token:
+            ap.error("--token is required (or set HT_TOKEN)")
+    elif not args.server or not args.token:
+        print("[dry-run] no --server/--token provided — skipping queue check, showing disk state only")
+
+    grid = json.loads(args.grid.read_text())
+    common_args = grid["common_args"]
+    api = _API(args.server, args.token) if (args.server and args.token) else None
+
+    # Fetch current server queue once (pending + running)
+    server_jobs: list[dict] = []
+    if api:
+        try:
+            all_jobs = api.get("/jobs")
+            server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")]
+            print(f"[server] {len(server_jobs)} job(s) currently pending/running in queue")
+        except Exception as e:
+            print(f"[warn] could not fetch server queue: {e}")
+
+    # Collect all runs: baseline + every group's runs
+    all_runs = [grid["baseline"]]
+    for group in grid["groups"]:
+        if group.get("needs_implementation"):
+            print(f"\n[skip] group '{group['name']}' — {group['needs_implementation']}")
+            continue
+        all_runs.extend(group["runs"])
+
+    submitted_total = 0
+    skipped_total   = 0
+
+    for run in all_runs:
+        run_name  = run["run_name"]
+        run_args  = common_args + run.get("extra_args", [])
+        done      = set(_completed_reps(run_name, args.reps))
+        queued    = _queued_reps(server_jobs, run_name)
+        accounted = done | queued
+        missing   = [i for i in range(args.reps) if i not in accounted]
+
+        if not missing:
+            if len(done) == args.reps:
+                print(f"\n[done]  {run_name}  ({args.reps}/{args.reps} reps complete)")
+            else:
+                in_q = sorted(queued - done)
+                print(f"\n[skip]  {run_name}  ({len(done)} done, {len(in_q)} queued: {[f'rep{i:02d}' for i in in_q]})")
+            skipped_total += 1
+            continue
+
+        parts = []
+        if done:    parts.append(f"{len(done)} done")
+        if queued:  parts.append(f"{len(queued - done)} queued")
+        status = ", ".join(parts) if parts else "not started"
+        print(f"\n[queue] {run_name}  ({status}) — submitting {len(missing)} rep(s)")
+        _submit_cv(api, run_name, run_args, args.reps, missing, args.dry_run)
+        submitted_total += len(missing)
+
+    print(f"\n{'='*50}")
+    print(f"Submitted: {submitted_total} jobs  |  Already accounted for: {skipped_total} runs")
+    if grid.get("groups"):
+        needs_impl = sum(1 for g in grid["groups"] if g.get("needs_implementation"))
+        if needs_impl:
+            print(f"Skipped (needs implementation): {needs_impl} group(s)")
+
+
+if __name__ == "__main__":
+    main()

v3/scripts/main/phase4/experiment_grid.json

@@ -0,0 +1,77 @@
+{
+  "_notes": [
+    "Phase 4 — Dual CNN architecture comparison (image only).",
+    "Goal: isolate the effect of bilateral processing by comparing architectures without clinical data.",
+    "Best settings from phase 3 carried forward: iop_ratio_drop_raw, bcd_p05 (p07 did not stack).",
+    "common_args are prepended to every run's args list.",
+    "All runs use --bridge-mode image_only — no clinical data."
+  ],
+
+  "common_args": [
+    "--eval-mode", "binary",
+    "--bridge-mode", "image_only",
+    "--epochs", "30",
+    "--in-memory-cache",
+    "--augment",
+    "--tune-binary-threshold",
+    "--backbone", "refugelike",
+    "--iop-corr-method", "ratio",
+    "--iop-drop-raw",
+    "--output-root", "v3/results"
+  ],
+
+  "_common_args_implicit_defaults": {
+    "--tower-loss-mode": "bcd",
+    "--bcd-prob": "0.5",
+    "--warmup-cd-epochs": "0",
+    "--bilat-warmup-tower-epochs": "3",
+    "--bilat-warmup-fused-epochs": "3"
+  },
+
+  "baseline": {
+    "run_name": "phase4/single",
+    "description": "Single-eye baseline with best phase 3 image settings. Direct comparison point for bilateral modes.",
+    "extra_args": ["--tower-mode", "single"]
+  },
+
+  "groups": [
+    {
+      "name": "architecture",
+      "description": "Core bilateral architecture comparison.",
+      "runs": [
+        {
+          "run_name": "phase4/ensemble",
+          "description": "Ensemble: two independent single-eye forward passes, patient-level average of OD+OS scores.",
+          "extra_args": ["--tower-mode", "ensemble"]
+        },
+        {
+          "run_name": "phase4/bilateral",
+          "description": "BilateralHT: shared towers, concat OD+OS → learned joint projection MLP → classifier.",
+          "extra_args": ["--tower-mode", "bilateral"]
+        },
+        {
+          "run_name": "phase4/siamese",
+          "description": "SiameseHT: shared backbone, mean+delta (asymmetry) representation → classifier.",
+          "extra_args": ["--tower-mode", "siamese"]
+        }
+      ]
+    },
+
+    {
+      "name": "loss_bilateral",
+      "description": "Test loss function sensitivity in bilateral modes (using winner from architecture group).",
+      "runs": [
+        {
+          "run_name": "phase4/bilateral_loss_all",
+          "description": "BilateralHT with all-losses mode — joint training dynamics may differ from single-eye.",
+          "extra_args": ["--tower-mode", "bilateral", "--tower-loss-mode", "all"]
+        },
+        {
+          "run_name": "phase4/siamese_loss_all",
+          "description": "SiameseHT with all-losses mode.",
+          "extra_args": ["--tower-mode", "siamese", "--tower-loss-mode", "all"]
+        }
+      ]
+    }
+  ]
+}

v3/scripts/main/phase5/dispatch_logit_mlp_ckpt.py

@@ -0,0 +1,170 @@
+"""
+Dispatch a 10×5 rep-CV of logit_mlp_head with --save-checkpoints.
+
+Results land in v3/results/phase5/logit_mlp_head_ckpt/{rep00..rep09}/binary/ensemble/
+
+Usage:
+    # Dry run
+    python -m v3.scripts.main.phase5.dispatch_logit_mlp_ckpt --dry-run
+
+    # Submit to server
+    python -m v3.scripts.main.phase5.dispatch_logit_mlp_ckpt \
+        --server http://hades:8765 --token hypertower
+
+    # Skip reps already done, re-queue only missing ones:
+    python -m v3.scripts.main.phase5.dispatch_logit_mlp_ckpt \
+        --server http://hades:8765 --token hypertower
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+import requests
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
+
+RUN_NAME       = "phase5/logit_mlp_head_ckpt"
+MODULE         = "v3.scripts.main.run_cv"
+OUTPUT_DIR     = "v3/results"
+RESULTS_ROOT   = Path(__file__).resolve().parents[4] / "v3" / "results"
+REP_SEED_START = 100
+REP_SEED_STEP  = 100
+N_REPS         = 10
+
+RUN_ARGS = [
+    "--eval-mode",            "binary",
+    "--bridge-mode",          "fused",
+    "--tower-mode",           "ensemble",
+    "--fused-head",
+    "--head-type",            "logit_mlp",
+    "--epochs",               "30",
+    "--in-memory-cache",
+    "--augment",
+    "--tune-binary-threshold",
+    "--backbone",             "refugelike",
+    "--iop-corr-method",      "ratio",
+    "--iop-drop-raw",
+    "--exclude-cols",         "Axial_Length",
+    "--output-root",          "v3/results",
+    "--save-checkpoints",
+]
+
+
+# ── Completion check ──────────────────────────────────────────────────────────
+
+def _completed_reps(reps: int) -> list[int]:
+    done = []
+    for i in range(reps):
+        rep_dir = RESULTS_ROOT / RUN_NAME / f"rep{i:02d}" / "binary" / "ensemble"
+        if (rep_dir / "summary.json").exists():
+            # Also verify at least one checkpoint exists
+            if any(rep_dir.glob("fold*/best_single.pt")):
+                done.append(i)
+            else:
+                print(f"  [warn] rep{i:02d} has summary.json but no checkpoints — will re-queue")
+    return done
+
+
+# ── Server API ────────────────────────────────────────────────────────────────
+
+class _API:
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self._h = {"x-token": token}
+
+    def get(self, path: str, **params) -> object:
+        r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+    def post(self, path: str, body: dict) -> dict:
+        r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+
+def _queued_reps(jobs: list[dict]) -> set[int]:
+    active = set()
+    for job in jobs:
+        if job.get("run_name") != RUN_NAME:
+            continue
+        if job["state"] not in ("pending", "running"):
+            continue
+        try:
+            args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"])
+            if "--rep-index" in args:
+                active.add(int(args[args.index("--rep-index") + 1]))
+        except Exception:
+            pass
+    return active
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                  formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--server",  default=os.environ.get("HT_SERVER", ""))
+    ap.add_argument("--token",   default=os.environ.get("HT_TOKEN",  ""))
+    ap.add_argument("--reps",    type=int, default=N_REPS)
+    ap.add_argument("--dry-run", action="store_true")
+    args = ap.parse_args()
+
+    if not args.dry_run:
+        if not args.server:
+            ap.error("--server required (or set HT_SERVER)")
+        if not args.token:
+            ap.error("--token required (or set HT_TOKEN)")
+
+    api = _API(args.server, args.token) if (args.server and args.token) else None
+
+    server_jobs: list[dict] = []
+    if api:
+        try:
+            all_jobs = api.get("/jobs")
+            server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")]
+            print(f"[server] {len(server_jobs)} job(s) pending/running")
+        except Exception as e:
+            print(f"[warn] could not fetch queue: {e}")
+
+    done   = set(_completed_reps(args.reps))
+    queued = _queued_reps(server_jobs)
+    missing = [i for i in range(args.reps) if i not in (done | queued)]
+
+    print(f"\nRun: {RUN_NAME}")
+    print(f"  Done:    {sorted(done)}")
+    print(f"  Queued:  {sorted(queued - done)}")
+    print(f"  Missing: {missing}")
+
+    if not missing:
+        print("Nothing to submit.")
+        return
+
+    for i in missing:
+        seed = REP_SEED_START + i * REP_SEED_STEP
+        rep_args = RUN_ARGS + [
+            "--run-name",       RUN_NAME,
+            "--reps",           "1",
+            "--rep-seed-start", str(seed),
+            "--rep-index",      str(i),
+        ]
+        body = {
+            "run_name":   RUN_NAME,
+            "module":     MODULE,
+            "args":       rep_args,
+            "output_dir": OUTPUT_DIR,
+            "priority":   0,
+        }
+        if args.dry_run:
+            print(f"  [dry-run] rep{i:02d}  seed={seed}")
+        else:
+            resp = api.post("/jobs", body)
+            print(f"  queued rep{i:02d}  seed={seed}  job_id={resp['job_id']}")
+
+
+if __name__ == "__main__":
+    main()

v3/scripts/main/phase5/dispatch_phase5.py

@@ -0,0 +1,199 @@
+"""
+Dispatch phase 5 experiment runs to the distributed job server.
+
+Reads experiment_grid.json, checks which runs already have complete 10x5 results,
+and submits the rest. Skips runs marked needs_implementation.
+
+Usage:
+    python -m v3.scripts.main.phase5.dispatch_phase5 \
+        --server http://hades:8765 --token hypertower
+
+    # Dry run (print what would be submitted, don't actually submit):
+    python -m v3.scripts.main.phase5.dispatch_phase5 \
+        --server http://hades:8765 --token hypertower --dry-run
+
+    # Override number of reps (default 10):
+    python -m v3.scripts.main.phase5.dispatch_phase5 \
+        --server http://hades:8765 --token hypertower --reps 4
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+import requests
+
+# Allow running as `python v3/scripts/main/phase3/dispatch_phase3.py`
+sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
+
+GRID_PATH   = Path(__file__).parent / "experiment_grid.json"
+RESULTS_ROOT = Path(__file__).resolve().parents[4] / "v3" / "results"
+MODULE      = "v3.scripts.main.run_cv"
+OUTPUT_DIR  = "v3/results"
+REP_SEED_START = 100
+REP_SEED_STEP  = 100
+
+
+# ── Completion check ──────────────────────────────────────────────────────────
+
+def _completed_reps(run_name: str, reps: int) -> list[int]:
+    """Return list of rep indices that already have a summary.json (any tower mode)."""
+    done = []
+    for i in range(reps):
+        rep_dir = RESULTS_ROOT / run_name / f"rep{i:02d}" / "binary"
+        # Accept any tower mode subdir
+        if rep_dir.exists() and any((rep_dir / tm / "summary.json").exists()
+                                     for tm in ("single", "bilateral", "siamese", "ensemble")):
+            done.append(i)
+    return done
+
+
+# ── Server API ────────────────────────────────────────────────────────────────
+
+class _API:
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self._h = {"x-token": token}
+
+    def get(self, path: str, **params) -> object:
+        r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+    def post(self, path: str, body: dict) -> dict:
+        r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+
+def _queued_reps(jobs: list[dict], run_name: str) -> set[int]:
+    """Return rep indices already pending or running in the server queue."""
+    active = set()
+    for job in jobs:
+        if job["run_name"] != run_name:
+            continue
+        if job["state"] not in ("pending", "running"):
+            continue
+        # Extract --rep-index from job args
+        try:
+            args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"])
+            if "--rep-index" in args:
+                active.add(int(args[args.index("--rep-index") + 1]))
+        except Exception:
+            pass
+    return active
+
+
+def _submit_cv(api: _API, run_name: str, run_args: list[str],
+               reps: int, missing: list[int], dry_run: bool):
+    """Submit one job per missing rep."""
+    for i in missing:
+        seed = REP_SEED_START + i * REP_SEED_STEP
+        rep_args = run_args + [
+            "--run-name", run_name,
+            "--reps", "1",
+            "--rep-seed-start", str(seed),
+            "--rep-index", str(i),
+        ]
+        body = {
+            "run_name": run_name,
+            "module": MODULE,
+            "args": rep_args,
+            "output_dir": OUTPUT_DIR,
+            "priority": 0,
+        }
+        if dry_run:
+            print(f"  [dry-run] would queue rep{i:02d}  seed={seed}")
+        else:
+            resp = api.post("/jobs", body)
+            print(f"  queued rep{i:02d}  seed={seed}  job_id={resp['job_id']}")
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                  formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""),
+                    help="Server URL (or set HT_SERVER)")
+    ap.add_argument("--token",  default=os.environ.get("HT_TOKEN",  ""),
+                    help="Shared secret (or set HT_TOKEN)")
+    ap.add_argument("--reps",   type=int, default=10,
+                    help="Expected number of reps per run (default: 10)")
+    ap.add_argument("--grid",   type=Path, default=GRID_PATH,
+                    help="Path to experiment grid JSON (default: experiment_grid.json)")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="Print what would be submitted without actually submitting")
+    args = ap.parse_args()
+
+    if not args.dry_run:
+        if not args.server:
+            ap.error("--server is required (or set HT_SERVER)")
+        if not args.token:
+            ap.error("--token is required (or set HT_TOKEN)")
+    elif not args.server or not args.token:
+        print("[dry-run] no --server/--token provided — skipping queue check, showing disk state only")
+
+    grid = json.loads(args.grid.read_text())
+    common_args = grid["common_args"]
+    api = _API(args.server, args.token) if (args.server and args.token) else None
+
+    # Fetch current server queue once (pending + running)
+    server_jobs: list[dict] = []
+    if api:
+        try:
+            all_jobs = api.get("/jobs")
+            server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")]
+            print(f"[server] {len(server_jobs)} job(s) currently pending/running in queue")
+        except Exception as e:
+            print(f"[warn] could not fetch server queue: {e}")
+
+    # Collect all runs: baseline + every group's runs
+    all_runs = [grid["baseline"]]
+    for group in grid["groups"]:
+        if group.get("needs_implementation"):
+            print(f"\n[skip] group '{group['name']}' — {group['needs_implementation']}")
+            continue
+        all_runs.extend(group["runs"])
+
+    submitted_total = 0
+    skipped_total   = 0
+
+    for run in all_runs:
+        run_name  = run["run_name"]
+        run_args  = common_args + run.get("extra_args", [])
+        done      = set(_completed_reps(run_name, args.reps))
+        queued    = _queued_reps(server_jobs, run_name)
+        accounted = done | queued
+        missing   = [i for i in range(args.reps) if i not in accounted]
+
+        if not missing:
+            if len(done) == args.reps:
+                print(f"\n[done]  {run_name}  ({args.reps}/{args.reps} reps complete)")
+            else:
+                in_q = sorted(queued - done)
+                print(f"\n[skip]  {run_name}  ({len(done)} done, {len(in_q)} queued: {[f'rep{i:02d}' for i in in_q]})")
+            skipped_total += 1
+            continue
+
+        parts = []
+        if done:    parts.append(f"{len(done)} done")
+        if queued:  parts.append(f"{len(queued - done)} queued")
+        status = ", ".join(parts) if parts else "not started"
+        print(f"\n[queue] {run_name}  ({status}) — submitting {len(missing)} rep(s)")
+        _submit_cv(api, run_name, run_args, args.reps, missing, args.dry_run)
+        submitted_total += len(missing)
+
+    print(f"\n{'='*50}")
+    print(f"Submitted: {submitted_total} jobs  |  Already accounted for: {skipped_total} runs")
+    if grid.get("groups"):
+        needs_impl = sum(1 for g in grid["groups"] if g.get("needs_implementation"))
+        if needs_impl:
+            print(f"Skipped (needs implementation): {needs_impl} group(s)")
+
+
+if __name__ == "__main__":
+    main()

v3/scripts/main/phase5/experiment_grid.json

@@ -0,0 +1,87 @@
+{
+  "_notes": [
+    "Phase 5 — Full HyperTower: bilateral + clinical data + aggregation strategy comparison.",
+    "Goal: show the effect of a dual CNN + clinical data, and compare ensemble vs fused-head.",
+    "Best settings from all prior phases: refugelike, iop_ratio_drop_raw, bcd_p05.",
+    "Best bilateral architecture from phase 4 should be used — update tower-mode accordingly.",
+    "common_args are prepended to every run's args list.",
+    "NOTE: update --tower-mode in groups below once phase 4 winner is known.",
+    "Placeholder uses 'bilateral' — change to 'siamese' if that wins phase 4."
+  ],
+
+  "common_args": [
+    "--eval-mode", "binary",
+    "--bridge-mode", "fused",
+    "--epochs", "30",
+    "--in-memory-cache",
+    "--augment",
+    "--tune-binary-threshold",
+    "--backbone", "refugelike",
+    "--iop-corr-method", "ratio",
+    "--iop-drop-raw",
+    "--exclude-cols", "Axial_Length",
+    "--output-root", "v3/results"
+  ],
+
+  "_common_args_implicit_defaults": {
+    "--tower-loss-mode": "bcd",
+    "--bcd-prob": "0.5",
+    "--warmup-cd-epochs": "40",
+    "--single-warmup-tower-epochs": "3",
+    "--single-warmup-fused-epochs": "3",
+    "--bilat-warmup-tower-epochs": "3",
+    "--bilat-warmup-fused-epochs": "3"
+  },
+
+  "baseline": {
+    "run_name": "phase5/single_fused",
+    "description": "Single-eye + clinical data — phase 3 best config, re-run as direct comparison baseline for phase 5.",
+    "extra_args": ["--tower-mode", "single"]
+  },
+
+  "groups": [
+    {
+      "name": "bilateral_clinical",
+      "description": "Add clinical data to bilateral architectures.",
+      "runs": [
+        {
+          "run_name": "phase5/ensemble_fused",
+          "description": "Ensemble (independent OD+OS) + clinical data via fused bridge.",
+          "extra_args": ["--tower-mode", "ensemble"]
+        },
+        {
+          "run_name": "phase5/bilateral_fused",
+          "description": "BilateralHT + clinical data — full canonical HyperTower.",
+          "extra_args": ["--tower-mode", "bilateral"]
+        },
+        {
+          "run_name": "phase5/siamese_fused",
+          "description": "SiameseHT + clinical data — siamese mean+delta with fused clinical bridge.",
+          "extra_args": ["--tower-mode", "siamese"]
+        }
+      ]
+    },
+
+    {
+      "name": "aggregation",
+      "description": "Compare patient-level prediction aggregation strategies on top of ensemble.",
+      "runs": [
+        {
+          "run_name": "phase5/ensemble_fused_head",
+          "description": "Ensemble + clinical data + attention scorer head (Linear(C→1) per eye, softmax-weighted average).",
+          "extra_args": ["--tower-mode", "ensemble", "--fused-head"]
+        },
+        {
+          "run_name": "phase5/logit_mlp_head",
+          "description": "Ensemble + clinical data + logit-level MLP head (cat([logit_od, logit_os]) → FC(64) → FC(C)).",
+          "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--head-type", "logit_mlp"]
+        },
+        {
+          "run_name": "phase5/embedding_mlp_head",
+          "description": "Ensemble + clinical data + embedding-level MLP head (cat([z_od, z_os]) → FC(256) → FC(C)).",
+          "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--head-type", "embedding_mlp"]
+        }
+      ]
+    }
+  ]
+}

v3/scripts/main/phase6/dispatch_phase6a.py

@@ -0,0 +1,204 @@
+"""
+Dispatch phase 6a experiment runs (geometry vector injection) to the distributed job server.
+
+Covers the geometry_vector_gt and geometry_vector_unet groups from experiment_grid.json:
+  - GT geometry vector × {single, ensemble, fused-head}
+  - U-Net geometry vector × {single, ensemble, fused-head}
+
+Skips geometry_tower_* groups (needs_implementation — will get dispatch_phase6b.py).
+
+Usage:
+    python -m v3.scripts.main.phase6.dispatch_phase6a \
+        --server http://hades:8765 --token hypertower
+
+    # Dry run (print what would be submitted, don't actually submit):
+    python -m v3.scripts.main.phase6.dispatch_phase6a \
+        --server http://hades:8765 --token hypertower --dry-run
+
+    # Override number of reps (default 10):
+    python -m v3.scripts.main.phase6.dispatch_phase6a \
+        --server http://hades:8765 --token hypertower --reps 4
+
+NOTE: Requires --geometry-dim and --geometry-source to be wired into
+v3_hypertower.py before these jobs will run successfully.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+import requests
+
+# Allow running as `python v3/scripts/main/phase6/dispatch_phase6a.py`
+sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
+
+GRID_PATH    = Path(__file__).parent / "experiment_grid.json"
+RESULTS_ROOT = Path(__file__).resolve().parents[4] / "v3" / "results"
+MODULE       = "v3.scripts.main.run_cv"
+OUTPUT_DIR   = "v3/results"
+REP_SEED_START = 100
+REP_SEED_STEP  = 100
+
+
+# ── Completion check ──────────────────────────────────────────────────────────
+
+def _completed_reps(run_name: str, reps: int) -> list[int]:
+    """Return list of rep indices that already have a summary.json (any tower mode)."""
+    done = []
+    for i in range(reps):
+        rep_dir = RESULTS_ROOT / run_name / f"rep{i:02d}" / "binary"
+        # Accept any tower mode subdir
+        if rep_dir.exists() and any((rep_dir / tm / "summary.json").exists()
+                                     for tm in ("single", "bilateral", "siamese", "ensemble", "tri", "tri_bilateral")):
+            done.append(i)
+    return done
+
+
+# ── Server API ────────────────────────────────────────────────────────────────
+
+class _API:
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self._h = {"x-token": token}
+
+    def get(self, path: str, **params) -> object:
+        r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+    def post(self, path: str, body: dict) -> dict:
+        r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+
+def _queued_reps(jobs: list[dict], run_name: str) -> set[int]:
+    """Return rep indices already pending or running in the server queue."""
+    active = set()
+    for job in jobs:
+        if job["run_name"] != run_name:
+            continue
+        if job["state"] not in ("pending", "running"):
+            continue
+        try:
+            args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"])
+            if "--rep-index" in args:
+                active.add(int(args[args.index("--rep-index") + 1]))
+        except Exception:
+            pass
+    return active
+
+
+def _submit_cv(api: _API, run_name: str, run_args: list[str],
+               reps: int, missing: list[int], dry_run: bool):
+    """Submit one job per missing rep."""
+    for i in missing:
+        seed = REP_SEED_START + i * REP_SEED_STEP
+        rep_args = run_args + [
+            "--run-name", run_name,
+            "--reps", "1",
+            "--rep-seed-start", str(seed),
+            "--rep-index", str(i),
+        ]
+        body = {
+            "run_name": run_name,
+            "module": MODULE,
+            "args": rep_args,
+            "output_dir": OUTPUT_DIR,
+            "priority": 0,
+        }
+        if dry_run:
+            print(f"  [dry-run] would queue rep{i:02d}  seed={seed}")
+        else:
+            resp = api.post("/jobs", body)
+            print(f"  queued rep{i:02d}  seed={seed}  job_id={resp['job_id']}")
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                  formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""),
+                    help="Server URL (or set HT_SERVER)")
+    ap.add_argument("--token",  default=os.environ.get("HT_TOKEN",  ""),
+                    help="Shared secret (or set HT_TOKEN)")
+    ap.add_argument("--reps",   type=int, default=10,
+                    help="Expected number of reps per run (default: 10)")
+    ap.add_argument("--grid",   type=Path, default=GRID_PATH,
+                    help="Path to experiment grid JSON (default: experiment_grid.json)")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="Print what would be submitted without actually submitting")
+    args = ap.parse_args()
+
+    if not args.dry_run:
+        if not args.server:
+            ap.error("--server is required (or set HT_SERVER)")
+        if not args.token:
+            ap.error("--token is required (or set HT_TOKEN)")
+    elif not args.server or not args.token:
+        print("[dry-run] no --server/--token provided — skipping queue check, showing disk state only")
+
+    grid = json.loads(args.grid.read_text())
+    common_args = grid["common_args"]
+    api = _API(args.server, args.token) if (args.server and args.token) else None
+
+    # Fetch current server queue once (pending + running)
+    server_jobs: list[dict] = []
+    if api:
+        try:
+            all_jobs = api.get("/jobs")
+            server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")]
+            print(f"[server] {len(server_jobs)} job(s) currently pending/running in queue")
+        except Exception as e:
+            print(f"[warn] could not fetch server queue: {e}")
+
+    # Collect all runs: baseline + every group's runs (skip needs_implementation)
+    all_runs = [grid["baseline"]]
+    for group in grid["groups"]:
+        if group.get("needs_implementation"):
+            print(f"\n[skip] group '{group['name']}' — {group['needs_implementation']}")
+            continue
+        all_runs.extend(group["runs"])
+
+    submitted_total = 0
+    skipped_total   = 0
+
+    for run in all_runs:
+        run_name  = run["run_name"]
+        run_args  = common_args + run.get("extra_args", [])
+        done      = set(_completed_reps(run_name, args.reps))
+        queued    = _queued_reps(server_jobs, run_name)
+        accounted = done | queued
+        missing   = [i for i in range(args.reps) if i not in accounted]
+
+        if not missing:
+            if len(done) == args.reps:
+                print(f"\n[done]  {run_name}  ({args.reps}/{args.reps} reps complete)")
+            else:
+                in_q = sorted(queued - done)
+                print(f"\n[skip]  {run_name}  ({len(done)} done, {len(in_q)} queued: {[f'rep{i:02d}' for i in in_q]})")
+            skipped_total += 1
+            continue
+
+        parts = []
+        if done:    parts.append(f"{len(done)} done")
+        if queued:  parts.append(f"{len(queued - done)} queued")
+        status = ", ".join(parts) if parts else "not started"
+        print(f"\n[queue] {run_name}  ({status}) — submitting {len(missing)} rep(s)")
+        _submit_cv(api, run_name, run_args, args.reps, missing, args.dry_run)
+        submitted_total += len(missing)
+
+    print(f"\n{'='*50}")
+    print(f"Submitted: {submitted_total} jobs  |  Already accounted for: {skipped_total} runs")
+    if grid.get("groups"):
+        needs_impl = sum(1 for g in grid["groups"] if g.get("needs_implementation"))
+        if needs_impl:
+            print(f"Skipped (needs implementation): {needs_impl} group(s)")
+
+
+if __name__ == "__main__":
+    main()

v3/scripts/main/phase6/experiment_grid.json

@@ -0,0 +1,144 @@
+{
+  "_notes": [
+    "Phase 6 — Geometry augmentation: vector injection and dedicated geometry tower.",
+    "Goal: test whether derived structural geometry (CDR, rim ratio, etc.) improves performance",
+    "  injected either as a 5-dim vector appended to the clinical stream (Part A)",
+    "  or as a dedicated geometry tower feeding into bridge fusion (Part B).",
+    "Tower modes under test: single, ensemble, fused-head (ensemble + --fused-head).",
+    "Geometry sources: GT contour annotations (no annotator-bias concern at this stage —",
+    "  labels were assigned by same clinicians, GT seg merely measures CDR directly)",
+    "  and U-Net segmentations (REFUGE-trained, fine-tuned on PAPILA folds — unbiased).",
+    "Part A (geometry_vector): vector injection — dispatch_phase6a.py covers this.",
+    "  Requires: --geometry-dim and --geometry-source wired into v3_hypertower.py.",
+    "Part B (geometry_tower): dedicated geometry tower — needs architecture implementation.",
+    "  Will get its own dispatch_phase6b.py once built.",
+    "Ensemble and fused-head baselines come from phase5. Single has no equivalent with all tuned",
+    "  hyperparameters, so a phase6 single baseline is included here.",
+    "Best settings from all prior phases: refugelike backbone, iop_ratio_drop_raw, bcd_p05.",
+    "Geometry features: [area_cdr, rim_ratio, vertical_cdr, horizontal_cdr, centre_shift] (dim=5).",
+    "common_args are prepended to every run's args list."
+  ],
+
+  "common_args": [
+    "--eval-mode", "binary",
+    "--bridge-mode", "fused",
+    "--epochs", "30",
+    "--in-memory-cache",
+    "--augment",
+    "--tune-binary-threshold",
+    "--backbone", "refugelike",
+    "--iop-corr-method", "ratio",
+    "--iop-drop-raw",
+    "--exclude-cols", "Axial_Length",
+    "--img-crop-manifest", "manifest.csv",
+    "--output-root", "v3/results"
+  ],
+
+  "_common_args_implicit_defaults": {
+    "--tower-loss-mode": "bcd",
+    "--bcd-prob": "0.5",
+    "--warmup-cd-epochs": "40",
+    "--single-warmup-tower-epochs": "3",
+    "--single-warmup-fused-epochs": "3",
+    "--bilat-warmup-tower-epochs": "3",
+    "--bilat-warmup-fused-epochs": "3"
+  },
+
+  "baseline": {
+    "run_name": "phase6/single_no_geom",
+    "description": "Single-eye + clinical data, no geometry — needed as Phase 6 baseline since no prior phase ran single with all tuned hyperparameters (iop_ratio_drop_raw, bcd_p05, refugelike).",
+    "extra_args": ["--tower-mode", "single"]
+  },
+
+  "groups": [
+    {
+      "name": "geometry_vector_gt",
+      "description": "Part A — Inject 5-dim geometry vector from GT annotations alongside clinical data. Three aggregation modes: single-eye, ensemble (independent OD+OS), ensemble with fused head.",
+      "runs": [
+        {
+          "run_name": "phase6/vec_gt_single",
+          "description": "Single-eye + clinical + GT geometry vector appended to clinical stream.",
+          "extra_args": ["--tower-mode", "single", "--geometry-dim", "5", "--geometry-source", "gt"]
+        },
+        {
+          "run_name": "phase6/vec_gt_ensemble",
+          "description": "Ensemble + clinical + GT geometry vector (per-eye geometry, independent OD+OS mean).",
+          "extra_args": ["--tower-mode", "ensemble", "--geometry-dim", "5", "--geometry-source", "gt"]
+        },
+        {
+          "run_name": "phase6/vec_gt_fused_head",
+          "description": "Ensemble + fused head + clinical + GT geometry vector.",
+          "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--geometry-dim", "5", "--geometry-source", "gt"]
+        }
+      ]
+    },
+
+    {
+      "name": "geometry_vector_unet",
+      "description": "Part A — Same three modes but geometry from U-Net segmentations (REFUGE-trained, fine-tuned). Tests whether GT annotator bias affects the geometry signal.",
+      "runs": [
+        {
+          "run_name": "phase6/vec_unet_single",
+          "description": "Single-eye + clinical + U-Net geometry vector.",
+          "extra_args": ["--tower-mode", "single", "--geometry-dim", "5", "--geometry-source", "unet"]
+        },
+        {
+          "run_name": "phase6/vec_unet_ensemble",
+          "description": "Ensemble + clinical + U-Net geometry vector.",
+          "extra_args": ["--tower-mode", "ensemble", "--geometry-dim", "5", "--geometry-source", "unet"]
+        },
+        {
+          "run_name": "phase6/vec_unet_fused_head",
+          "description": "Ensemble + fused head + clinical + U-Net geometry vector.",
+          "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--geometry-dim", "5", "--geometry-source", "unet"]
+        }
+      ]
+    },
+
+    {
+      "name": "geometry_tower_gt",
+      "description": "Part B — Dedicated geometry tower (MLP on 5-dim vector, separate from clinical tower) with GT geometry. Requires tri-tower bridge architecture.",
+      "needs_implementation": "Geometry tower not yet built — requires GeometryTower MLP, bridge reconfiguration for n>=3 towers, and --geometry-tower CLI flag.",
+      "runs": [
+        {
+          "run_name": "phase6/tower_gt_single",
+          "description": "Single-eye + clinical tower + dedicated GT geometry tower.",
+          "extra_args": ["--tower-mode", "single", "--geometry-tower", "--geometry-source", "gt"]
+        },
+        {
+          "run_name": "phase6/tower_gt_ensemble",
+          "description": "Ensemble + clinical tower + dedicated GT geometry tower.",
+          "extra_args": ["--tower-mode", "ensemble", "--geometry-tower", "--geometry-source", "gt"]
+        },
+        {
+          "run_name": "phase6/tower_gt_fused_head",
+          "description": "Ensemble + fused head + clinical tower + dedicated GT geometry tower.",
+          "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--geometry-tower", "--geometry-source", "gt"]
+        }
+      ]
+    },
+
+    {
+      "name": "geometry_tower_unet",
+      "description": "Part B — Same three modes with dedicated geometry tower, U-Net geometry source.",
+      "needs_implementation": "Geometry tower not yet built — requires GeometryTower MLP, bridge reconfiguration for n>=3 towers, and --geometry-tower CLI flag.",
+      "runs": [
+        {
+          "run_name": "phase6/tower_unet_single",
+          "description": "Single-eye + clinical tower + dedicated U-Net geometry tower.",
+          "extra_args": ["--tower-mode", "single", "--geometry-tower", "--geometry-source", "unet"]
+        },
+        {
+          "run_name": "phase6/tower_unet_ensemble",
+          "description": "Ensemble + clinical tower + dedicated U-Net geometry tower.",
+          "extra_args": ["--tower-mode", "ensemble", "--geometry-tower", "--geometry-source", "unet"]
+        },
+        {
+          "run_name": "phase6/tower_unet_fused_head",
+          "description": "Ensemble + fused head + clinical tower + dedicated U-Net geometry tower.",
+          "extra_args": ["--tower-mode", "ensemble", "--fused-head", "--geometry-tower", "--geometry-source", "unet"]
+        }
+      ]
+    }
+  ]
+}

v3/scripts/main/run_cv.py

@@ -54,6 +54,11 @@ def build_parser() -> argparse.ArgumentParser:
         "--rep-seed-step", type=int, default=100,
         help="Increment between rep fold-seeds (default: 100; rep k uses seed start + k*step).",
     )
+    ap.add_argument(
+        "--rep-index", type=int, default=None,
+        help="Override the rep directory index (e.g. 3 → rep03). "
+             "Used by the distributed server to run a single rep of a multi-rep job.",
+    )
     return ap
 
 
@@ -65,22 +70,24 @@ def main():
     seed_start     = int(args.rep_seed_start)
     seed_step      = int(args.rep_seed_step)
     base_run_name  = args.run_name or "v3_cv"
+    rep_index_override = getattr(args, "rep_index", None)
 
     for rep in range(reps):
         rep_seed = seed_start + rep * seed_step
         args.fold_seed = rep_seed
 
-        if reps > 1:
-            args.run_name = f"{base_run_name}/rep{rep:02d}"
+        dir_index = rep_index_override if (rep_index_override is not None and reps == 1) else rep
+        if reps > 1 or rep_index_override is not None:
+            args.run_name = f"{base_run_name}/rep{dir_index:02d}"
             print(f"\n{'='*60}", flush=True)
-            print(f"Rep {rep+1}/{reps}  fold_seed={rep_seed}", flush=True)
+            print(f"Rep {dir_index+1}  fold_seed={rep_seed}", flush=True)
             print(f"{'='*60}", flush=True)
         else:
             args.run_name = base_run_name
 
         tower = V3HyperTower(args)
         out_dir = tower.run()
-        print(f"\nRep {rep+1} output: {out_dir}", flush=True)
+        print(f"\nRep {dir_index+1} output: {out_dir}", flush=True)
 
 
 if __name__ == "__main__":