pre-refactor 041426
This commit is contained in:
rpotter6298
@@ -0,0 +1,170 @@
|
||||
"""
|
||||
Dispatch a 10×5 rep-CV of logit_mlp_head with --save-checkpoints.
|
||||
|
||||
Results land in v3/results/phase5/logit_mlp_head_ckpt/{rep00..rep09}/binary/ensemble/
|
||||
|
||||
Usage:
|
||||
# Dry run
|
||||
python -m v3.scripts.main.phase5.dispatch_logit_mlp_ckpt --dry-run
|
||||
|
||||
# Submit to server
|
||||
python -m v3.scripts.main.phase5.dispatch_logit_mlp_ckpt \
|
||||
--server http://hades:8765 --token hypertower
|
||||
|
||||
# Skip reps already done, re-queue only missing ones:
|
||||
python -m v3.scripts.main.phase5.dispatch_logit_mlp_ckpt \
|
||||
--server http://hades:8765 --token hypertower
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
|
||||
|
||||
RUN_NAME = "phase5/logit_mlp_head_ckpt"
|
||||
MODULE = "v3.scripts.main.run_cv"
|
||||
OUTPUT_DIR = "v3/results"
|
||||
RESULTS_ROOT = Path(__file__).resolve().parents[4] / "v3" / "results"
|
||||
REP_SEED_START = 100
|
||||
REP_SEED_STEP = 100
|
||||
N_REPS = 10
|
||||
|
||||
RUN_ARGS = [
|
||||
"--eval-mode", "binary",
|
||||
"--bridge-mode", "fused",
|
||||
"--tower-mode", "ensemble",
|
||||
"--fused-head",
|
||||
"--head-type", "logit_mlp",
|
||||
"--epochs", "30",
|
||||
"--in-memory-cache",
|
||||
"--augment",
|
||||
"--tune-binary-threshold",
|
||||
"--backbone", "refugelike",
|
||||
"--iop-corr-method", "ratio",
|
||||
"--iop-drop-raw",
|
||||
"--exclude-cols", "Axial_Length",
|
||||
"--output-root", "v3/results",
|
||||
"--save-checkpoints",
|
||||
]
|
||||
|
||||
|
||||
# ── Completion check ──────────────────────────────────────────────────────────
|
||||
|
||||
def _completed_reps(reps: int) -> list[int]:
|
||||
done = []
|
||||
for i in range(reps):
|
||||
rep_dir = RESULTS_ROOT / RUN_NAME / f"rep{i:02d}" / "binary" / "ensemble"
|
||||
if (rep_dir / "summary.json").exists():
|
||||
# Also verify at least one checkpoint exists
|
||||
if any(rep_dir.glob("fold*/best_single.pt")):
|
||||
done.append(i)
|
||||
else:
|
||||
print(f" [warn] rep{i:02d} has summary.json but no checkpoints — will re-queue")
|
||||
return done
|
||||
|
||||
|
||||
# ── Server API ────────────────────────────────────────────────────────────────
|
||||
|
||||
class _API:
|
||||
def __init__(self, base_url: str, token: str):
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self._h = {"x-token": token}
|
||||
|
||||
def get(self, path: str, **params) -> object:
|
||||
r = requests.get(f"{self.base_url}{path}", headers=self._h, params=params, timeout=10)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def post(self, path: str, body: dict) -> dict:
|
||||
r = requests.post(f"{self.base_url}{path}", headers=self._h, json=body, timeout=10)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
def _queued_reps(jobs: list[dict]) -> set[int]:
|
||||
active = set()
|
||||
for job in jobs:
|
||||
if job.get("run_name") != RUN_NAME:
|
||||
continue
|
||||
if job["state"] not in ("pending", "running"):
|
||||
continue
|
||||
try:
|
||||
args = job["args"] if isinstance(job["args"], list) else json.loads(job["args"])
|
||||
if "--rep-index" in args:
|
||||
active.add(int(args[args.index("--rep-index") + 1]))
|
||||
except Exception:
|
||||
pass
|
||||
return active
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""))
|
||||
ap.add_argument("--token", default=os.environ.get("HT_TOKEN", ""))
|
||||
ap.add_argument("--reps", type=int, default=N_REPS)
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not args.dry_run:
|
||||
if not args.server:
|
||||
ap.error("--server required (or set HT_SERVER)")
|
||||
if not args.token:
|
||||
ap.error("--token required (or set HT_TOKEN)")
|
||||
|
||||
api = _API(args.server, args.token) if (args.server and args.token) else None
|
||||
|
||||
server_jobs: list[dict] = []
|
||||
if api:
|
||||
try:
|
||||
all_jobs = api.get("/jobs")
|
||||
server_jobs = [j for j in all_jobs if j["state"] in ("pending", "running")]
|
||||
print(f"[server] {len(server_jobs)} job(s) pending/running")
|
||||
except Exception as e:
|
||||
print(f"[warn] could not fetch queue: {e}")
|
||||
|
||||
done = set(_completed_reps(args.reps))
|
||||
queued = _queued_reps(server_jobs)
|
||||
missing = [i for i in range(args.reps) if i not in (done | queued)]
|
||||
|
||||
print(f"\nRun: {RUN_NAME}")
|
||||
print(f" Done: {sorted(done)}")
|
||||
print(f" Queued: {sorted(queued - done)}")
|
||||
print(f" Missing: {missing}")
|
||||
|
||||
if not missing:
|
||||
print("Nothing to submit.")
|
||||
return
|
||||
|
||||
for i in missing:
|
||||
seed = REP_SEED_START + i * REP_SEED_STEP
|
||||
rep_args = RUN_ARGS + [
|
||||
"--run-name", RUN_NAME,
|
||||
"--reps", "1",
|
||||
"--rep-seed-start", str(seed),
|
||||
"--rep-index", str(i),
|
||||
]
|
||||
body = {
|
||||
"run_name": RUN_NAME,
|
||||
"module": MODULE,
|
||||
"args": rep_args,
|
||||
"output_dir": OUTPUT_DIR,
|
||||
"priority": 0,
|
||||
}
|
||||
if args.dry_run:
|
||||
print(f" [dry-run] rep{i:02d} seed={seed}")
|
||||
else:
|
||||
resp = api.post("/jobs", body)
|
||||
print(f" queued rep{i:02d} seed={seed} job_id={resp['job_id']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user