hypertower/v3/scripts/main/phase5/dispatch_logit_mlp_ckpt.py

"""
Dispatch a 10×5 rep-CV of logit_mlp_head with --save-checkpoints.

Results land in v3/results/phase5/logit_mlp_head_ckpt/{rep00..rep09}/binary/ensemble/

Usage:
    # Dry run
    python -m v3.scripts.main.phase5.dispatch_logit_mlp_ckpt --dry-run

    # Submit to server
    python -m v3.scripts.main.phase5.dispatch_logit_mlp_ckpt \
        --server http://hades:8765 --token hypertower

    # Skip reps already done, re-queue only missing ones:
    python -m v3.scripts.main.phase5.dispatch_logit_mlp_ckpt \
        --server http://hades:8765 --token hypertower
"""
from __future__ import annotations

import argparse
import json
import os
import sys
from pathlib import Path

import requests

sys.path.insert(0, str(Path(__file__).resolve().parents[4]))

RUN_NAME       = "phase5/logit_mlp_head_ckpt"
MODULE         = "v3.scripts.main.run_cv"
OUTPUT_DIR     = "v3/results"
RESULTS_ROOT   = Path(__file__).resolve().parents[4] / "v3" / "results"
REP_SEED_START = 100
REP_SEED_STEP  = 100
N_REPS         = 10

RUN_ARGS = [
    "--eval-mode",            "binary",
    "--bridge-mode",          "fused",
    "--tower-mode",           "ensemble",
    "--fused-head",
    "--head-type",            "logit_mlp",
    "--epochs",               "30",
    "--in-memory-cache",
    "--augment",
    "--tune-binary-threshold",
    "--backbone",             "refugelike",
    "--iop-corr-method",      "ratio",
    "--iop-drop-raw",
    "--exclude-cols",         "Axial_Length",
    "--output-root",          "v3/results",
    "--save-checkpoints",
]


# ── Completion check ──────────────────────────────────────────────────────────

def _completed_reps(reps: int) -> list[int]:
    done = []
    for i in range(reps):
        rep_dir = RESULTS_ROOT / RUN_NAME / f"rep{i:02d}" / "binary" / "ensemble"
        if (rep_dir / "summary.json").exists():
            # Also verify at least one checkpoint exists
            if any(rep_dir.glob("fold*/best_single.pt")):
                done.append(i)
            else:
                print(f"  [warn] rep{i:02d} has summary.json but no checkpoints — will re-queue")
    return done


# ── Server API ────────────────────────────────────────────────────────────────

class _API:
    def __init__(self, base_url: str, token: str):
        self.base_url = base_url.rstrip("/")
        self._h = {"x-token": token}

    def get(self, path: str, **params) -> object:
        r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10)
        r.raise_for_status()
        return r.json()

    def post(self, path: str, body: dict) -> dict:
        r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10)
        r.raise_for_status()
        return r.json()


def _queued_reps(jobs: list[dict]) -> set[int]:
    active = set()
    for job in jobs:
        if job.get("run_name") != RUN_NAME:
            continue
        if job["state"] not in ("pending", "running"):
            continue
        try:
            args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"])
            if "--rep-index" in args:
                active.add(int(args[args.index("--rep-index") + 1]))
        except Exception:
            pass
    return active


# ── Main ──────────────────────────────────────────────────────────────────────

def main():
    ap = argparse.ArgumentParser(description=__doc__,
                                  formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--server",  default=os.environ.get("HT_SERVER", ""))
    ap.add_argument("--token",   default=os.environ.get("HT_TOKEN",  ""))
    ap.add_argument("--reps",    type=int, default=N_REPS)
    ap.add_argument("--dry-run", action="store_true")
    args = ap.parse_args()

    if not args.dry_run:
        if not args.server:
            ap.error("--server required (or set HT_SERVER)")
        if not args.token:
            ap.error("--token required (or set HT_TOKEN)")

    api = _API(args.server, args.token) if (args.server and args.token) else None

    server_jobs: list[dict] = []
    if api:
        try:
            all_jobs = api.get("/jobs")
            server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")]
            print(f"[server] {len(server_jobs)} job(s) pending/running")
        except Exception as e:
            print(f"[warn] could not fetch queue: {e}")

    done   = set(_completed_reps(args.reps))
    queued = _queued_reps(server_jobs)
    missing = [i for i in range(args.reps) if i not in (done | queued)]

    print(f"\nRun: {RUN_NAME}")
    print(f"  Done:    {sorted(done)}")
    print(f"  Queued:  {sorted(queued - done)}")
    print(f"  Missing: {missing}")

    if not missing:
        print("Nothing to submit.")
        return

    for i in missing:
        seed = REP_SEED_START + i * REP_SEED_STEP
        rep_args = RUN_ARGS + [
            "--run-name",       RUN_NAME,
            "--reps",           "1",
            "--rep-seed-start", str(seed),
            "--rep-index",      str(i),
        ]
        body = {
            "run_name":   RUN_NAME,
            "module":     MODULE,
            "args":       rep_args,
            "output_dir": OUTPUT_DIR,
            "priority":   0,
        }
        if args.dry_run:
            print(f"  [dry-run] rep{i:02d}  seed={seed}")
        else:
            resp = api.post("/jobs", body)
            print(f"  queued rep{i:02d}  seed={seed}  job_id={resp['job_id']}")


if __name__ == "__main__":
    main()