pre-refactor 041426

This commit is contained in:
rpotter6298
2026-04-14 19:42:16 +02:00
parent eb9eafe715
commit 13290575d5
75 changed files with 8900 additions and 77 deletions
View File
+197
View File
@@ -0,0 +1,197 @@
"""
Dispatch phase 3 experiment runs to the distributed job server.
Reads experiment_grid.json, checks which runs already have complete 10x5 results,
and submits the rest via submit-cv. Skips runs marked needs_implementation.
Usage:
python -m v3.scripts.main.phase3.dispatch_phase3 \
--server http://hades:8765 --token hypertower
# Dry run (print what would be submitted, don't actually submit):
python -m v3.scripts.main.phase3.dispatch_phase3 \
--server http://hades:8765 --token hypertower --dry-run
# Override number of reps (default 10):
python -m v3.scripts.main.phase3.dispatch_phase3 \
--server http://hades:8765 --token hypertower --reps 4
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
import requests
# Allow running as `python v3/scripts/main/phase3/dispatch_phase3.py`
sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
GRID_PATH = Path(__file__).parent / "experiment_grid.json"
RESULTS_ROOT = Path(__file__).resolve().parents[4] / "v3" / "results"
MODULE = "v3.scripts.main.run_cv"
OUTPUT_DIR = "v3/results"
REP_SEED_START = 100
REP_SEED_STEP = 100
# ── Completion check ──────────────────────────────────────────────────────────
def _completed_reps(run_name: str, reps: int) -> list[int]:
"""Return list of rep indices that already have a summary.json."""
done = []
for i in range(reps):
summary = RESULTS_ROOT / run_name / f"rep{i:02d}" / "binary" / "single" / "summary.json"
if summary.exists():
done.append(i)
return done
# ── Server API ────────────────────────────────────────────────────────────────
class _API:
def __init__(self, base_url: str, token: str):
self.base_url = base_url.rstrip("/")
self._h = {"x-token": token}
def get(self, path: str, **params) -> object:
r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10)
r.raise_for_status()
return r.json()
def post(self, path: str, body: dict) -> dict:
r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10)
r.raise_for_status()
return r.json()
def _queued_reps(jobs: list[dict], run_name: str) -> set[int]:
"""Return rep indices already pending or running in the server queue."""
active = set()
for job in jobs:
if job["run_name"] != run_name:
continue
if job["state"] not in ("pending", "running"):
continue
# Extract --rep-index from job args
try:
args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"])
if "--rep-index" in args:
active.add(int(args[args.index("--rep-index") + 1]))
except Exception:
pass
return active
def _submit_cv(api: _API, run_name: str, run_args: list[str],
reps: int, missing: list[int], dry_run: bool):
"""Submit one job per missing rep."""
for i in missing:
seed = REP_SEED_START + i * REP_SEED_STEP
rep_args = run_args + [
"--run-name", run_name,
"--reps", "1",
"--rep-seed-start", str(seed),
"--rep-index", str(i),
]
body = {
"run_name": run_name,
"module": MODULE,
"args": rep_args,
"output_dir": OUTPUT_DIR,
"priority": 0,
}
if dry_run:
print(f" [dry-run] would queue rep{i:02d} seed={seed}")
else:
resp = api.post("/jobs", body)
print(f" queued rep{i:02d} seed={seed} job_id={resp['job_id']}")
# ── Main ──────────────────────────────────────────────────────────────────────
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""),
help="Server URL (or set HT_SERVER)")
ap.add_argument("--token", default=os.environ.get("HT_TOKEN", ""),
help="Shared secret (or set HT_TOKEN)")
ap.add_argument("--reps", type=int, default=10,
help="Expected number of reps per run (default: 10)")
ap.add_argument("--grid", type=Path, default=GRID_PATH,
help="Path to experiment grid JSON (default: experiment_grid.json)")
ap.add_argument("--dry-run", action="store_true",
help="Print what would be submitted without actually submitting")
args = ap.parse_args()
if not args.dry_run:
if not args.server:
ap.error("--server is required (or set HT_SERVER)")
if not args.token:
ap.error("--token is required (or set HT_TOKEN)")
elif not args.server or not args.token:
print("[dry-run] no --server/--token provided — skipping queue check, showing disk state only")
grid = json.loads(args.grid.read_text())
common_args = grid["common_args"]
api = _API(args.server, args.token) if (args.server and args.token) else None
# Fetch current server queue once (pending + running)
server_jobs: list[dict] = []
if api:
try:
all_jobs = api.get("/jobs")
server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")]
print(f"[server] {len(server_jobs)} job(s) currently pending/running in queue")
except Exception as e:
print(f"[warn] could not fetch server queue: {e}")
# Collect all runs: baseline + every group's runs
all_runs = [grid["baseline"]]
for group in grid["groups"]:
if group.get("needs_implementation"):
print(f"\n[skip] group '{group['name']}'{group['needs_implementation']}")
continue
all_runs.extend(group["runs"])
submitted_total = 0
skipped_total = 0
for run in all_runs:
run_name = run["run_name"]
run_args = common_args + run.get("extra_args", [])
done = set(_completed_reps(run_name, args.reps))
queued = _queued_reps(server_jobs, run_name)
accounted = done | queued
missing = [i for i in range(args.reps) if i not in accounted]
if not missing:
if len(done) == args.reps:
print(f"\n[done] {run_name} ({args.reps}/{args.reps} reps complete)")
else:
in_q = sorted(queued - done)
print(f"\n[skip] {run_name} ({len(done)} done, {len(in_q)} queued: {[f'rep{i:02d}' for i in in_q]})")
skipped_total += 1
continue
parts = []
if done: parts.append(f"{len(done)} done")
if queued: parts.append(f"{len(queued - done)} queued")
status = ", ".join(parts) if parts else "not started"
print(f"\n[queue] {run_name} ({status}) — submitting {len(missing)} rep(s)")
_submit_cv(api, run_name, run_args, args.reps, missing, args.dry_run)
submitted_total += len(missing)
print(f"\n{'='*50}")
print(f"Submitted: {submitted_total} jobs | Already accounted for: {skipped_total} runs")
if grid.get("groups"):
needs_impl = sum(1 for g in grid["groups"] if g.get("needs_implementation"))
if needs_impl:
print(f"Skipped (needs implementation): {needs_impl} group(s)")
if __name__ == "__main__":
main()
+66
View File
@@ -0,0 +1,66 @@
{
"_notes": [
"Epoch length sensitivity experiments.",
"All other settings match the phase3 baseline (fused bridge, BCD p=0.5, refugelike, etc.).",
"common_args are prepended to every run's args list."
],
"common_args": [
"--eval-mode", "binary",
"--tower-mode", "single",
"--in-memory-cache",
"--augment",
"--tune-binary-threshold",
"--backbone", "refugelike",
"--output-root", "v3/results"
],
"_common_args_implicit_defaults": {
"--bridge-mode": "fused",
"--tower-loss-mode": "bcd",
"--bcd-prob": "0.5",
"--warmup-cd-epochs": "40",
"--single-warmup-tower-epochs": "3",
"--single-warmup-fused-epochs": "3"
},
"baseline": {
"run_name": "phase3/epochs_30",
"description": "30-epoch run — same as phase3 baseline, included here for direct comparison.",
"extra_args": ["--epochs", "30"]
},
"groups": [
{
"name": "epoch_length",
"description": "Test sensitivity to total training epochs (warmup epochs unchanged).",
"runs": [
{
"run_name": "phase3/epochs_1",
"description": "1 epoch total — effectively pure warmup output with a single main-phase step.",
"extra_args": ["--epochs", "1"]
},
{
"run_name": "phase3/epochs_5",
"description": "5 epochs total.",
"extra_args": ["--epochs", "5"]
},
{
"run_name": "phase3/epochs_10",
"description": "10 epochs total.",
"extra_args": ["--epochs", "10"]
},
{
"run_name": "phase3/epochs_20",
"description": "20 epochs total.",
"extra_args": ["--epochs", "20"]
},
{
"run_name": "phase3/epochs_50",
"description": "50 epochs total.",
"extra_args": ["--epochs", "50"]
}
]
}
]
}
+362
View File
@@ -0,0 +1,362 @@
{
"_notes": [
"All runs use single-eye tower mode, binary eval, 10x5 rep-CV.",
"common_args are prepended to every run's args list.",
"Entries marked 'needs_implementation' require small code changes before running (noted inline).",
"Bridge fusion uses elementwise product of projected image/clinical features.",
"SE infrastructure exists in Bridge/ImageTower/ClinicalTower but use_se is hardcoded False",
" in SingleEyeHT \u2014 add --se-img-tower / --se-cd-tower / --se-bridge flags to wire through.",
"Dropout is hardcoded: Bridge classifier=0.5, ClinicalTower=0.1 \u2014 add --bridge-dropout /",
" --cd-dropout flags to make configurable."
],
"common_args": [
"--eval-mode",
"binary",
"--tower-mode",
"single",
"--epochs",
"30",
"--in-memory-cache",
"--augment",
"--tune-binary-threshold",
"--backbone",
"refugelike",
"--output-root",
"v3/results"
],
"_common_args_implicit_defaults": {
"--bridge-mode": "fused",
"--tower-loss-mode": "bcd",
"--bcd-prob": "0.5",
"--warmup-cd-epochs": "40",
"--single-warmup-tower-epochs": "3",
"--single-warmup-fused-epochs": "3"
},
"baseline": {
"run_name": "phase3/baseline",
"description": "Fused bridge (image+clinical), BCD p=0.5, cd_warmup=40, tower/fused warmup=3/3, no SE, no IOP correction.",
"extra_args": []
},
"groups": [
{
"name": "loss_function",
"description": "Test BCD loss variants vs cross-entropy baseline.",
"runs": [
{
"run_name": "phase3/loss_all",
"description": "All-losses mode (cross-entropy on all three heads every step).",
"extra_args": [
"--tower-loss-mode",
"all"
]
},
{
"run_name": "phase3/loss_bcd_p03",
"description": "BCD with lower switching probability (more CE, less BCD).",
"extra_args": [
"--tower-loss-mode",
"bcd",
"--bcd-prob",
"0.3"
]
},
{
"run_name": "phase3/loss_bcd_p07",
"description": "BCD with higher switching probability (more BCD, less CE).",
"extra_args": [
"--tower-loss-mode",
"bcd",
"--bcd-prob",
"0.7"
]
}
]
},
{
"name": "se_attention",
"description": "Squeeze-and-excitation gates at different points in the network.",
"runs": [
{
"run_name": "phase3/se_bridge",
"description": "SE gate on fused vector inside the bridge only.",
"extra_args": [
"--se-bridge"
]
},
{
"run_name": "phase3/se_img_tower",
"description": "SE gate on image tower output features.",
"extra_args": [
"--se-img-tower"
]
},
{
"run_name": "phase3/se_cd_tower",
"description": "SE gate on clinical tower output features.",
"extra_args": [
"--se-cd-tower"
]
},
{
"run_name": "phase3/se_all",
"description": "SE gates on image tower, clinical tower, and bridge.",
"extra_args": [
"--se-img-tower",
"--se-cd-tower",
"--se-bridge"
]
}
]
},
{
"name": "iop_correction",
"description": "Test different IOP measurement correction strategies (default: no correction).",
"runs": [
{
"run_name": "phase3/iop_ratio",
"description": "IOP correction via Perkins\u2192Pneumatic ratio scaling.",
"extra_args": [
"--iop-corr-method",
"ratio"
]
},
{
"run_name": "phase3/iop_ols",
"description": "IOP correction via OLS regression.",
"extra_args": [
"--iop-corr-method",
"ols"
]
},
{
"run_name": "phase3/iop_lad",
"description": "IOP correction via LAD (robust to outliers) regression.",
"extra_args": [
"--iop-corr-method",
"lad"
]
},
{
"run_name": "phase3/iop_multi",
"description": "IOP correction via multivariate regression including pachymetry.",
"extra_args": [
"--iop-corr-method",
"multi"
]
},
{
"run_name": "phase3/iop_ratio_drop_raw",
"description": "Ratio correction + drop raw IOP (only corrected IOP seen by model).",
"extra_args": [
"--iop-corr-method",
"ratio",
"--iop-drop-raw"
]
}
]
},
{
"name": "feature_ablation",
"description": "Exclude individual clinical features to measure each one's contribution.",
"runs": [
{
"run_name": "phase3/excl_iop",
"description": "No IOP features \u2014 tests how much intraocular pressure contributes.",
"extra_args": [
"--exclude-cols",
"IOP",
"Pachymetry"
]
},
{
"run_name": "phase3/excl_age",
"description": "No age feature.",
"extra_args": [
"--exclude-cols",
"Age"
]
},
{
"run_name": "phase3/excl_axial_length",
"description": "No axial length feature.",
"extra_args": [
"--exclude-cols",
"Axial_Length"
]
},
{
"run_name": "phase3/excl_refractive",
"description": "No refractive defect feature.",
"extra_args": [
"--exclude-cols",
"Refractive_Defect"
]
}
]
},
{
"name": "network_dims",
"description": "Test sensitivity to clinical tower and bridge fusion dimensionality.",
"runs": [
{
"run_name": "phase3/cd_hidden_64",
"description": "Smaller clinical tower (64 hidden units vs default 128).",
"extra_args": [
"--cd-hidden-dim",
"64"
]
},
{
"run_name": "phase3/cd_hidden_256",
"description": "Larger clinical tower (256 hidden units vs default 128).",
"extra_args": [
"--cd-hidden-dim",
"256"
]
},
{
"run_name": "phase3/fusion_dim_128",
"description": "Smaller fusion space (128 vs default 256).",
"extra_args": [
"--fusion-dim",
"128"
]
},
{
"run_name": "phase3/fusion_dim_512",
"description": "Larger fusion space (512 vs default 256).",
"extra_args": [
"--fusion-dim",
"512"
]
}
]
},
{
"name": "backbone_freezing",
"description": "Partial backbone freezing to reduce overfitting and speed training.",
"runs": [
{
"run_name": "phase3/freeze_25",
"description": "Freeze earliest 25% of backbone blocks.",
"extra_args": [
"--freeze-ratio",
"0.25"
]
},
{
"run_name": "phase3/freeze_50",
"description": "Freeze earliest 50% of backbone blocks.",
"extra_args": [
"--freeze-ratio",
"0.50"
]
}
]
},
{
"name": "learning_rate",
"description": "Test LR sensitivity (default 1e-4).",
"runs": [
{
"run_name": "phase3/lr_1e3",
"description": "Higher learning rate 1e-3.",
"extra_args": [
"--lr",
"1e-3"
]
},
{
"run_name": "phase3/lr_3e4",
"description": "Intermediate learning rate 3e-4.",
"extra_args": [
"--lr",
"3e-4"
]
},
{
"run_name": "phase3/lr_1e5",
"description": "Lower learning rate 1e-5.",
"extra_args": [
"--lr",
"1e-5"
]
}
]
},
{
"name": "dropout",
"description": "Test bridge classifier and clinical tower dropout rates.",
"runs": [
{
"run_name": "phase3/bridge_dropout_03",
"description": "Reduce bridge classifier dropout from 0.5 to 0.3.",
"extra_args": [
"--bridge-dropout",
"0.3"
]
},
{
"run_name": "phase3/bridge_dropout_07",
"description": "Increase bridge classifier dropout to 0.7.",
"extra_args": [
"--bridge-dropout",
"0.7"
]
},
{
"run_name": "phase3/cd_dropout_03",
"description": "Increase clinical tower dropout from 0.1 to 0.3.",
"extra_args": [
"--cd-dropout",
"0.3"
]
}
]
},
{
"name": "warmup",
"description": "Test warmup ablations vs default (cd=40, tower/fused=3/3).",
"runs": [
{
"run_name": "phase3/warmup_no_cd",
"description": "No CD warmup (cd=0) \u2014 tests whether the 40-epoch CD warmup is necessary.",
"extra_args": [
"--warmup-cd-epochs",
"0"
]
},
{
"run_name": "phase3/warmup_tower5_fused5",
"description": "Extended tower/fused warmup (5/5 vs default 3/3).",
"extra_args": [
"--single-warmup-tower-epochs",
"5",
"--single-warmup-fused-epochs",
"5"
]
}
]
},
{
"name": "sampling_augmentation",
"description": "Test data sampling and augmentation choices.",
"runs": [
{
"run_name": "phase3/no_augment",
"description": "No augmentation \u2014 baseline images only.",
"extra_args": [
"--no-augment"
]
},
{
"run_name": "phase3/balanced_sampling",
"description": "Weighted balanced sampler to counter class imbalance.",
"extra_args": [
"--balanced-sampling"
]
}
]
}
]
}
+54
View File
@@ -0,0 +1,54 @@
{
"_notes": [
"Phase 3.5 — confirmation and BCD tuning.",
"All runs use best settings from phase 3: refugelike backbone, ratio IOP correction, drop raw IOP, exclude axial length.",
"Baseline here is phase3/iop_ratio_drop_raw (0.8685 ± 0.011) — already complete, not re-run.",
"common_args are prepended to every run's args list."
],
"common_args": [
"--eval-mode", "binary",
"--tower-mode", "single",
"--epochs", "30",
"--in-memory-cache",
"--augment",
"--tune-binary-threshold",
"--backbone", "refugelike",
"--iop-corr-method", "ratio",
"--iop-drop-raw",
"--exclude-cols", "Axial_Length",
"--output-root", "v3/results"
],
"_common_args_implicit_defaults": {
"--bridge-mode": "fused",
"--warmup-cd-epochs": "40",
"--single-warmup-tower-epochs": "3",
"--single-warmup-fused-epochs": "3"
},
"baseline": {
"run_name": "phase35/iop_bcd_p07",
"description": "Best IOP preprocessing + best BCD prob from phase 3 combined.",
"extra_args": ["--tower-loss-mode", "bcd", "--bcd-prob", "0.7"]
},
"groups": [
{
"name": "bcd_tuning",
"description": "Extended BCD probability sweep with best IOP settings.",
"runs": [
{
"run_name": "phase35/iop_bcd_p08",
"description": "BCD p=0.8 with ratio IOP + drop raw.",
"extra_args": ["--tower-loss-mode", "bcd", "--bcd-prob", "0.8"]
},
{
"run_name": "phase35/iop_bcd_p09",
"description": "BCD p=0.9 with ratio IOP + drop raw.",
"extra_args": ["--tower-loss-mode", "bcd", "--bcd-prob", "0.9"]
}
]
}
]
}