Add distributed server implementation and protocol definitions

- Introduced `protocol.py` for shared data models used in server/client communication, including request and response schemas for registration, job submission, and status updates.
- Implemented `server.py` to manage a SQLite job queue and client registry, handling job polling, status updates, and job completion.
- Created a cheat sheet for server usage, detailing commands for starting the server, submitting jobs, and monitoring clients.
- Added several experiment configuration files for various training setups, including geometry vector injections and baseline ensembles.

v4/distributed/batch_dispatch.py

@@ -0,0 +1,287 @@
+"""
+v4 batch dispatch — merge a batch.json with a base config and submit jobs.
+
+Each batch entry specifies only what differs from the base config. Each entry
+is dispatched as N_REPS independent jobs (full 5-fold CV per rep, different seeds),
+writing to {run_name}/rep00/, rep01/, etc.
+
+Usage:
+    python -m v4.distributed.batch_dispatch \\
+        --server http://apollo:8765 \\
+        --token <secret> \\
+        --config v4/configs/ensemble_fused.json \\
+        --batch v4/scripts/experiments/my_batch.json \\
+        [--reps 10] \\
+        [--seed-start 1234] \\
+        [--seed-step 100] \\
+        [--fold-seed-start 100] \\
+        [--fold-seed-step 100] \\
+        [--output-root v4/results] \\
+        [--priority 0] \\
+        [--dry-run]
+
+batch.json format:
+    [
+      {
+        "run_name":  "experiments/lr_sweep/lr1e3",   // required
+        "overrides": {                               // optional — deep-merged into base
+          "training": { "lr": 0.001 }
+        },
+        "stage_overrides": {                         // optional — patched by stage name
+          "nt": { "epochs": 40 }
+        },
+        "reps":     10,                              // optional — overrides --reps
+        "priority": 0                                // optional
+      }
+    ]
+
+Global flags (can also be set via env vars):
+    --server  HT_SERVER
+    --token   HT_TOKEN
+"""
+from __future__ import annotations
+
+import argparse
+import copy
+import json
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import requests
+
+# ──────────────────────────────────────────────────────────────
+# Config helpers
+# ──────────────────────────────────────────────────────────────
+
+def deep_merge(base: dict, override: dict) -> dict:
+    """Recursively merge override into a copy of base.
+
+    - Dicts are merged recursively.
+    - All other types (scalars, lists) are replaced by the override value.
+    """
+    result = copy.deepcopy(base)
+    for k, v in override.items():
+        if k in result and isinstance(result[k], dict) and isinstance(v, dict):
+            result[k] = deep_merge(result[k], v)
+        else:
+            result[k] = copy.deepcopy(v)
+    return result
+
+
+def apply_stage_overrides(stages: list[dict], stage_overrides: dict) -> list[dict]:
+    """Patch individual stages by name without replacing the entire list."""
+    stages = copy.deepcopy(stages)
+    for stage in stages:
+        name = stage.get("name")
+        if name in stage_overrides:
+            merged = deep_merge(stage, stage_overrides[name])
+            stage.clear()
+            stage.update(merged)
+    return stages
+
+
+def build_config(base_cfg: dict, entry: dict, rep: int, seed: int, fold_seed: int,
+                 output_root: str) -> dict:
+    """Produce the final merged config for one rep of one batch entry."""
+    cfg = copy.deepcopy(base_cfg)
+
+    # Deep-merge top-level overrides
+    cfg = deep_merge(cfg, entry.get("overrides", {}))
+
+    # Patch individual stages by name
+    if "stage_overrides" in entry and "stages" in cfg:
+        cfg["stages"] = apply_stage_overrides(cfg["stages"], entry["stage_overrides"])
+
+    # Stamp run_name, model seed, split seed, output_root.
+    base_run_name = entry["run_name"]
+    cfg["run_name"]    = f"{base_run_name}/rep{rep:02d}"
+    cfg["seed"]        = seed
+    cfg["fold_seed"]   = fold_seed
+    cfg["output_root"] = output_root
+
+    return cfg
+
+
+def save_dispatched_config(cfg: dict, base_run_name: str, rep: int,
+                           repo_root: Path) -> Path:
+    """Write the merged config to v4/configs/dispatched/ and return its path."""
+    out_dir = repo_root / "v4" / "configs" / "dispatched" / base_run_name
+    out_dir.mkdir(parents=True, exist_ok=True)
+    path = out_dir / f"rep{rep:02d}.json"
+    path.write_text(json.dumps(cfg, indent=2))
+    return path
+
+
+# ──────────────────────────────────────────────────────────────
+# Server HTTP helper
+# ──────────────────────────────────────────────────────────────
+
+class _API:
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self._h = {"x-token": token}
+
+    def post(self, path: str, body: dict) -> dict:
+        r = requests.post(f"{self.base_url}{path}", headers=self._h,
+                          json=body, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+
+# ──────────────────────────────────────────────────────────────
+# Dispatch
+# ──────────────────────────────────────────────────────────────
+
+def dispatch_batch(
+    api: _API,
+    base_cfg: dict,
+    batch: list[dict],
+    *,
+    default_reps:    int,
+    seed_start:      int,
+    seed_step:       int,
+    fold_seed_start: int,
+    fold_seed_step:  int,
+    output_root:     str,
+    default_priority: int,
+    server_path:     str,
+    repo_root:       Path,
+    dry_run:         bool,
+) -> None:
+    total = skipped = 0
+    for entry in batch:
+        run_name = entry["run_name"]
+        reps     = entry.get("reps", default_reps)
+        priority = entry.get("priority", default_priority)
+
+        print(f"\n[dispatch] {run_name}  ({reps} reps)")
+
+        for rep in range(reps):
+            seed      = seed_start + rep * seed_step
+            fold_seed = fold_seed_start + rep * fold_seed_step
+            cfg       = build_config(base_cfg, entry, rep, seed, fold_seed, output_root)
+            cfg_path = save_dispatched_config(cfg, run_name, rep, repo_root)
+
+            rel_path        = cfg_path.relative_to(repo_root)
+            server_cfg_path = str(rel_path)
+
+            job_body = {
+                "run_name":   run_name,
+                "module":     "v4.classes.v4_hypertower",
+                "args":       ["--config", server_cfg_path],
+                "output_dir": output_root,
+                "priority":   priority,
+            }
+
+            if dry_run:
+                print(
+                    f"  [dry-run] rep{rep:02d}  seed={seed}  "
+                    f"fold_seed={fold_seed}  config={server_cfg_path}"
+                )
+                total += 1
+            else:
+                resp = api.post("/jobs", job_body)
+                if resp.get("skipped"):
+                    print(f"  rep{rep:02d}  [skip — results exist on disk]")
+                    skipped += 1
+                elif resp.get("duplicate"):
+                    print(f"  rep{rep:02d}  [skip — already queued]  job_id={resp['job_id']}")
+                    skipped += 1
+                else:
+                    print(
+                        f"  rep{rep:02d}  seed={seed}  fold_seed={fold_seed}  "
+                        f"job_id={resp['job_id']}  config={server_cfg_path}"
+                    )
+                    total += 1
+
+    action = "would submit" if dry_run else "submitted"
+    skip_note = f"  ({skipped} already queued/done, skipped)" if skipped else ""
+    print(f"\n[dispatch] {action} {total} jobs across {len(batch)} experiment(s){skip_note}")
+
+
+# ──────────────────────────────────────────────────────────────
+# Entry point
+# ──────────────────────────────────────────────────────────────
+
+def main():
+    repo_root = Path(__file__).resolve().parents[2]
+
+    ap = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""),
+                    help="Server URL (or set HT_SERVER)")
+    ap.add_argument("--token", default=os.environ.get("HT_TOKEN", ""),
+                    help="Shared secret (or set HT_TOKEN)")
+    ap.add_argument("--config", required=True,
+                    help="Base config JSON file")
+    ap.add_argument("--batch", required=True,
+                    help="Batch JSON file listing experiments")
+    ap.add_argument("--reps", type=int, default=10,
+                    help="Repetitions per experiment (default: 10)")
+    ap.add_argument("--seed-start", type=int, default=1234,
+                    help="Seed for rep00 (default: 1234)")
+    ap.add_argument("--seed-step", type=int, default=100,
+                    help="Seed increment per rep (default: 100)")
+    ap.add_argument("--fold-seed-start", type=int, default=100,
+                    help="Split fold_seed for rep00 (default: 100; matches v3)")
+    ap.add_argument("--fold-seed-step", type=int, default=100,
+                    help="Split fold_seed increment per rep (default: 100; matches v3)")
+    ap.add_argument("--output-root", default="v4/results",
+                    help="Output root written into each config (default: v4/results)")
+    ap.add_argument("--server-path", default="",
+                    help="Absolute path to hypertower root on server "
+                         "(used to build config paths in job args; "
+                         "if omitted, relative paths are used)")
+    ap.add_argument("--priority", type=int, default=0,
+                    help="Default job priority (default: 0)")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="Print jobs without submitting")
+    args = ap.parse_args()
+
+    if not args.dry_run:
+        if not args.server:
+            ap.error("--server is required (or set HT_SERVER)")
+        if not args.token:
+            ap.error("--token is required (or set HT_TOKEN)")
+
+    cfg_path = Path(args.config)
+    if not cfg_path.is_absolute():
+        cfg_path = repo_root / cfg_path
+    base_cfg = json.loads(cfg_path.read_text())
+
+    batch_path = Path(args.batch)
+    if not batch_path.is_absolute():
+        batch_path = repo_root / batch_path
+    batch = json.loads(batch_path.read_text())
+
+    if not isinstance(batch, list):
+        sys.exit("batch.json must be a JSON array")
+    for i, entry in enumerate(batch):
+        if "run_name" not in entry:
+            sys.exit(f"batch entry {i} is missing required 'run_name'")
+
+    api = _API(args.server, args.token) if not args.dry_run else None
+
+    dispatch_batch(
+        api,
+        base_cfg,
+        batch,
+        default_reps=args.reps,
+        seed_start=args.seed_start,
+        seed_step=args.seed_step,
+        fold_seed_start=args.fold_seed_start,
+        fold_seed_step=args.fold_seed_step,
+        output_root=args.output_root,
+        default_priority=args.priority,
+        server_path=args.server_path,
+        repo_root=repo_root,
+        dry_run=args.dry_run,
+    )
+
+
+if __name__ == "__main__":
+    main()

v4/distributed/cli.py

@@ -0,0 +1,347 @@
+"""
+HyperTower distributed job CLI — submit jobs, view status.
+
+Usage:
+    # View all connected clients
+    python -m v4.distributed.cli clients
+
+    # View a specific client
+    python -m v4.distributed.cli clients <client_id>
+
+    # Live monitoring
+    python -m v4.distributed.cli clients --watch
+
+    # View jobs (optionally filter by state)
+    python -m v4.distributed.cli jobs [--state pending|running|done|failed]
+
+    # Submit a job
+    python -m v4.distributed.cli submit \\
+        --run-name v4/ensemble_fused \\
+        -- --config v4/configs/ensemble_fused.json
+
+    # Submit all jobs from a batch file (JSON)
+    python -m v4.distributed.cli submit-batch jobs.json
+
+    # Cancel a pending job
+    python -m v4.distributed.cli cancel <job_id>
+
+Global flags (can also be set via env vars):
+    --server  HT_SERVER   e.g. http://apollo:8765
+    --token   HT_TOKEN
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import time
+from datetime import datetime, timezone
+from typing import Optional
+
+import requests
+
+# ──────────────────────────────────────────────────────────────
+# HTTP helpers
+# ──────────────────────────────────────────────────────────────
+
+class _API:
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self._h = {"x-token": token}
+
+    def get(self, path: str, **params) -> object:
+        r = requests.get(f"{self.base_url}{path}", headers=self._h,
+                         params=params, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+    def post(self, path: str, body: dict) -> object:
+        r = requests.post(f"{self.base_url}{path}", headers=self._h,
+                          json=body, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+    def delete(self, path: str) -> object:
+        r = requests.delete(f"{self.base_url}{path}", headers=self._h, timeout=10)
+        r.raise_for_status()
+        return r.json()
+
+
+# ──────────────────────────────────────────────────────────────
+# Formatting helpers
+# ──────────────────────────────────────────────────────────────
+
+def _ago(ts: Optional[str]) -> str:
+    if not ts:
+        return "-"
+    try:
+        dt = datetime.fromisoformat(ts)
+        delta = datetime.now(timezone.utc) - dt
+        secs = int(delta.total_seconds())
+        if secs < 60:
+            return f"{secs}s ago"
+        elif secs < 3600:
+            return f"{secs//60}m ago"
+        else:
+            return f"{secs//3600}h{(secs%3600)//60}m ago"
+    except Exception:
+        return ts
+
+
+def _table(rows: list[list[str]], headers: list[str]):
+    widths = [max(len(str(r[i])) for r in ([headers] + rows)) for i in range(len(headers))]
+    sep = "  "
+    def _row(r):
+        return sep.join(str(r[i]).ljust(widths[i]) for i in range(len(r)))
+    print(_row(headers))
+    print("-" * (sum(widths) + len(sep) * (len(widths) - 1)))
+    for r in rows:
+        print(_row(r))
+
+
+# ──────────────────────────────────────────────────────────────
+# Subcommands
+# ──────────────────────────────────────────────────────────────
+
+def _clients_table(api: _API) -> str:
+    clients = api.get("/clients")
+    if not clients:
+        return "No clients connected."
+    rows = []
+    for c in clients:
+        s = c["status"]
+        parts = []
+        if s.get("fold") is not None:
+            parts.append(f"fold{s['fold']}")
+        if s.get("stage"):
+            parts.append(s["stage"])
+        if s.get("epoch") is not None:
+            parts.append(f"ep{s['epoch']}/{s.get('total_epochs', '?')}")
+        if s.get("last_val_auc") is not None:
+            parts.append(f"auc={s['last_val_auc']:.4f}")
+        prog = " ".join(parts) if parts else "-"
+        rows.append([
+            c["client_id"],
+            c["hostname"],
+            c["gpu_info"][:30],
+            s["state"],
+            s.get("run_name") or "-",
+            prog,
+            _ago(c["last_seen"]),
+        ])
+    headers = ["ID", "HOST", "GPU", "STATE", "RUN", "PROGRESS", "SEEN"]
+    widths = [max(len(str(r[i])) for r in ([headers] + rows)) for i in range(len(headers))]
+    sep = "  "
+    lines = []
+    lines.append(sep.join(str(h).ljust(widths[i]) for i, h in enumerate(headers)))
+    lines.append("-" * (sum(widths) + len(sep) * (len(widths) - 1)))
+    for r in rows:
+        lines.append(sep.join(str(r[i]).ljust(widths[i]) for i in range(len(r))))
+    return "\n".join(lines)
+
+
+def cmd_clients(api: _API, args):
+    if hasattr(args, "client_id") and args.client_id:
+        data = api.get(f"/clients/{args.client_id}")
+        s = data["status"]
+        print(f"client_id  : {data['client_id']}")
+        print(f"hostname   : {data['hostname']}")
+        print(f"gpu        : {data['gpu_info']}")
+        print(f"last_seen  : {_ago(data['last_seen'])}")
+        print(f"state      : {s['state']}")
+        if s.get("job_id"):
+            print(f"job        : {s['job_id']}  ({s.get('run_name', '')})")
+        if s.get("fold") is not None:
+            print(f"progress   : fold {s['fold']}  stage {s.get('stage', '?')}  "
+                  f"ep {s.get('epoch', '?')}/{s.get('total_epochs', '?')}  "
+                  f"val_auc={s.get('last_val_auc', '?')}")
+        return
+
+    watch = getattr(args, "watch", False)
+    interval = getattr(args, "interval", 5)
+
+    if not watch:
+        print(_clients_table(api))
+        return
+
+    try:
+        while True:
+            now = datetime.now().strftime("%H:%M:%S")
+            print(f"\033[H\033[2J", end="")
+            print(f"HyperTower clients  [{now}]  (Ctrl-C to exit)\n")
+            print(_clients_table(api))
+            time.sleep(interval)
+    except KeyboardInterrupt:
+        print("\nStopped.")
+
+
+def cmd_jobs(api: _API, args):
+    params = {}
+    if hasattr(args, "state") and args.state:
+        params["state"] = args.state
+    jobs = api.get("/jobs", **params)
+    if not jobs:
+        print("No jobs.")
+        return
+
+    try:
+        clients = api.get("/clients")
+        id_to_client = {c["client_id"]: c for c in clients}
+    except Exception:
+        id_to_client = {}
+
+    rows = []
+    for j in jobs:
+        attempts  = j.get("attempts", 0)
+        client_id = j.get("assigned_to")
+        client    = id_to_client.get(client_id) if client_id else None
+
+        client_label = (client["hostname"] if client else client_id) if client_id else "-"
+
+        progress = "-"
+        if client:
+            s     = client.get("status", {})
+            parts = []
+            if s.get("fold") is not None:
+                parts.append(f"fold{s['fold']}")
+            if s.get("stage"):
+                parts.append(s["stage"])
+            if s.get("epoch") is not None:
+                parts.append(f"ep{s['epoch']}/{s.get('total_epochs', '?')}")
+            if parts:
+                progress = " ".join(parts)
+
+        rows.append([
+            j["job_id"][:12],
+            j["run_name"],
+            j["state"],
+            f"{attempts}" if attempts else "-",
+            client_label,
+            progress,
+            _ago(j["created_at"]),
+            _ago(j.get("started_at")),
+            _ago(j.get("completed_at")),
+        ])
+    _table(rows, ["JOB_ID", "RUN_NAME", "STATE", "TRIES", "CLIENT", "PROGRESS", "CREATED", "STARTED", "DONE"])
+    pending = sum(1 for j in jobs if j["state"] == "pending")
+    running = sum(1 for j in jobs if j["state"] == "running")
+    done    = sum(1 for j in jobs if j["state"] == "done")
+    failed  = sum(1 for j in jobs if j["state"] == "failed")
+    print(f"\n  {len(jobs)} total  |  {pending} pending  {running} running  {done} done  {failed} failed")
+
+
+def cmd_submit(api: _API, args):
+    body = {
+        "run_name": args.run_name,
+        "module": args.module,
+        "args": args.run_args,
+        "output_dir": args.output_dir,
+        "priority": args.priority,
+    }
+    resp = api.post("/jobs", body)
+    print(f"Queued job {resp['job_id']} ({args.run_name})")
+
+
+def cmd_submit_batch(api: _API, args):
+    with open(args.batch_file) as f:
+        jobs = json.load(f)
+    for job in jobs:
+        resp = api.post("/jobs", job)
+        print(f"Queued {resp['job_id']}  ({job['run_name']})")
+
+
+def cmd_cancel(api: _API, args):
+    resp = api.delete(f"/jobs/{args.job_id}")
+    print(f"Cancelled {args.job_id}" if resp.get("ok") else resp)
+
+
+def cmd_clear(api: _API, args):
+    body: dict = {}
+    if args.all:
+        body["all"] = True
+    elif args.run_name:
+        body["run_name"] = args.run_name
+    else:
+        body["states"] = args.states or ["done", "failed", "cancelled"]
+    resp = api.post("/jobs/clear", body)
+    print(f"Cleared {resp['cleared']} jobs.")
+
+
+# ──────────────────────────────────────────────────────────────
+# Parser
+# ──────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                  formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--server", default=os.environ.get("HT_SERVER", ""),
+                    help="Server URL (or set HT_SERVER)")
+    ap.add_argument("--token", default=os.environ.get("HT_TOKEN", ""),
+                    help="Shared secret (or set HT_TOKEN)")
+
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    # clients
+    p_cl = sub.add_parser("clients", help="List clients or inspect one")
+    p_cl.add_argument("client_id", nargs="?")
+    p_cl.add_argument("--watch", "-w", action="store_true",
+                      help="Live monitoring mode — refresh every --interval seconds")
+    p_cl.add_argument("--interval", "-n", type=int, default=5,
+                      help="Refresh interval in seconds for --watch (default: 5)")
+
+    # jobs
+    p_j = sub.add_parser("jobs", help="List jobs")
+    p_j.add_argument("--state", choices=["pending", "running", "done", "failed", "cancelled"])
+
+    # submit
+    p_s = sub.add_parser("submit", help="Submit a single job")
+    p_s.add_argument("--run-name", required=True)
+    p_s.add_argument("--module", default="v4.classes.v4_hypertower")
+    p_s.add_argument("--output-dir", default="v4/results")
+    p_s.add_argument("--priority", type=int, default=0)
+    p_s.add_argument("run_args", nargs=argparse.REMAINDER,
+                     help="Args after '--' are forwarded to the module")
+
+    # submit-batch
+    p_b = sub.add_parser("submit-batch", help="Submit jobs from a JSON file")
+    p_b.add_argument("batch_file")
+
+    # cancel
+    p_c = sub.add_parser("cancel", help="Cancel a pending job")
+    p_c.add_argument("job_id")
+
+    # clear
+    p_cl2 = sub.add_parser("clear", help="Delete jobs by run-name, state, or everything")
+    p_cl2.add_argument("--run-name", default=None, help="Delete all jobs with this run-name")
+    p_cl2.add_argument("--states", nargs="+",
+                       default=None,
+                       choices=["done", "failed", "cancelled", "pending", "running"],
+                       help="Delete jobs in these states (default: done+failed+cancelled)")
+    p_cl2.add_argument("--all", action="store_true", help="Delete ALL jobs")
+
+    args = ap.parse_args()
+
+    if not args.server:
+        ap.error("--server is required (or set HT_SERVER)")
+    if not args.token:
+        ap.error("--token is required (or set HT_TOKEN)")
+
+    if hasattr(args, "run_args") and args.run_args and args.run_args[0] == "--":
+        args.run_args = args.run_args[1:]
+
+    api = _API(args.server, args.token)
+
+    dispatch = {
+        "clients": cmd_clients,
+        "jobs": cmd_jobs,
+        "submit": cmd_submit,
+        "submit-batch": cmd_submit_batch,
+        "cancel": cmd_cancel,
+        "clear": cmd_clear,
+    }
+    dispatch[args.cmd](api, args)
+
+
+if __name__ == "__main__":
+    main()

v4/distributed/client.py

@@ -0,0 +1,407 @@
+"""
+HyperTower distributed job client daemon.
+
+Registers with the server, polls for jobs, syncs code, runs training,
+uploads results, and loops. Parses stdout to stream live status.
+
+Usage:
+    python -m v4.distributed.client \\
+        --server http://apollo:8765 \\
+        --token <secret> \\
+        --server-ssh rpotter@apollo \\
+        --server-path /home/rpotter/hypertower \\
+        [--local-path ~/hypertower] \\
+        [--poll-interval 15]
+
+Compatibility test (verify GPU env, 1-fold dry-run):
+    python -m v4.distributed.client ... --test --config v4/configs/ensemble_fused.json
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import shutil
+import signal
+import socket
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+from typing import Optional
+
+import requests
+
+from .protocol import (
+    JobResult,
+    JobSpec,
+    PollResponse,
+    RegisterRequest,
+    RegisterResponse,
+    StatusPush,
+)
+
+# ──────────────────────────────────────────────────────────────
+# Server HTTP wrapper
+# ──────────────────────────────────────────────────────────────
+
+class _Server:
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self._h = {"x-token": token}
+        self.client_id: str = ""
+
+    def _post(self, path: str, **kw) -> dict:
+        r = requests.post(f"{self.base_url}{path}", headers=self._h, timeout=15, **kw)
+        r.raise_for_status()
+        return r.json()
+
+    def register(self, hostname: str, gpu_info: str) -> str:
+        data = self._post("/register",
+                          json={"hostname": hostname, "gpu_info": gpu_info})
+        self.client_id = data["client_id"]
+        self.hostname = hostname
+        self.gpu_info = gpu_info
+        return self.client_id
+
+    def _reregister(self):
+        try:
+            self._post("/register",
+                       json={"hostname": self.hostname, "gpu_info": self.gpu_info},
+                       params={"reuse_id": self.client_id})
+        except Exception:
+            pass
+
+    def poll(self) -> Optional[JobSpec]:
+        data = self._post("/poll", params={"client_id": self.client_id})
+        if data.get("please_reregister"):
+            self._reregister()
+        return JobSpec(**data["job"]) if data.get("job") else None
+
+    def push_status(self, status: StatusPush):
+        try:
+            r = requests.post(
+                f"{self.base_url}/status/{self.client_id}",
+                json=status.model_dump(),
+                headers=self._h,
+                timeout=5,
+            )
+            if r.ok and r.json().get("please_reregister"):
+                self._reregister()
+        except Exception:
+            pass  # don't crash job on status push failure
+
+    def complete(self, job_id: str, success: bool, error_msg: Optional[str] = None):
+        self._post("/complete",
+                   json={"job_id": job_id, "success": success, "error_msg": error_msg})
+
+
+# ──────────────────────────────────────────────────────────────
+# GPU info
+# ──────────────────────────────────────────────────────────────
+
+def _gpu_info() -> str:
+    # NVIDIA
+    try:
+        out = subprocess.check_output(
+            ["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader"],
+            text=True, stderr=subprocess.DEVNULL,
+        ).strip()
+        if out:
+            return " | ".join(out.splitlines())
+    except Exception:
+        pass
+    # AMD
+    try:
+        out = subprocess.check_output(
+            ["rocm-smi", "--showproductname", "--csv"],
+            text=True, stderr=subprocess.DEVNULL,
+        ).strip().splitlines()
+        names = [l for l in out if l and not l.startswith("device")]
+        if names:
+            return "AMD: " + " | ".join(names)
+    except Exception:
+        pass
+    # AMD fallback
+    try:
+        out = subprocess.check_output(
+            ["rocminfo"],
+            text=True, stderr=subprocess.DEVNULL,
+        )
+        names = [l.split(":", 1)[1].strip() for l in out.splitlines()
+                 if "Marketing Name:" in l]
+        if names:
+            return "AMD: " + " | ".join(names)
+    except Exception:
+        pass
+    return "no-gpu"
+
+
+# ──────────────────────────────────────────────────────────────
+# rsync helpers
+# ──────────────────────────────────────────────────────────────
+
+def _rsync(src: str, dst: str, delete: bool = False):
+    cmd = ["rsync", "-az", "--info=progress2"]
+    if delete:
+        cmd.append("--delete")
+    cmd += [src, dst]
+    subprocess.run(cmd, check=True)
+
+
+def _sync_code(server_ssh: str, server_path: str, local_path: str):
+    """Pull v4/ source from server → local (overwrites local changes)."""
+    src = f"{server_ssh}:{server_path}/v4/"
+    dst = f"{local_path}/v4/"
+    Path(dst).mkdir(parents=True, exist_ok=True)
+    _rsync(src, dst, delete=True)
+
+
+def _upload_results(server_ssh: str, server_path: str, local_path: str,
+                    run_name: str, output_dir: str):
+    src = f"{local_path}/{output_dir}/{run_name}/"
+    dst = f"{server_ssh}:{server_path}/{output_dir}/{run_name}/"
+    remote_parent = f"{server_path}/{output_dir}/{Path(run_name).parent}"
+    subprocess.run(["ssh", server_ssh, f"mkdir -p '{remote_parent}'"], check=True)
+    _rsync(src, dst)
+
+
+def _clean_local(local_path: str, run_name: str, output_dir: str):
+    target = Path(local_path) / output_dir / run_name
+    if target.exists():
+        shutil.rmtree(target)
+        print(f"[client] cleaned {target}", flush=True)
+
+
+# ──────────────────────────────────────────────────────────────
+# Stdout parsers (match v4_hypertower.py / fusion.py print format)
+# ──────────────────────────────────────────────────────────────
+
+# "  fold2 [nt] ep004/036 [fused_warmup  ]  loss=0.4321  acc=0.876  val_auc=0.7654"
+_EP_RE = re.compile(
+    r"fold(\d+)\s+\[([^\]]+)\]\s+ep(\d+)/(\d+).*?val_auc=([0-9.nan]+)",
+    re.IGNORECASE,
+)
+# "── fold 2/5  train_groups=..."
+_FOLD_RE = re.compile(r"fold\s+(\d+)/\d+", re.IGNORECASE)
+
+
+def _parse_line(line: str) -> dict:
+    """Return any structured fields found in a stdout line."""
+    out = {}
+    m = _FOLD_RE.search(line)
+    if m:
+        out["fold"] = int(m.group(1))
+    m = _EP_RE.search(line)
+    if m:
+        out["fold"]         = int(m.group(1))
+        out["stage"]        = m.group(2)
+        out["epoch"]        = int(m.group(3))
+        out["total_epochs"] = int(m.group(4))
+        try:
+            out["last_val_auc"] = float(m.group(5))
+        except ValueError:
+            pass
+    return out
+
+
+# ──────────────────────────────────────────────────────────────
+# Core job runner
+# ──────────────────────────────────────────────────────────────
+
+def _run_job(job: JobSpec, server: _Server,
+             server_ssh: str, server_path: str, local_path: str,
+             no_sync: bool = False,
+             extra_args: list[str] | None = None) -> bool:
+    extra_args = extra_args or []
+    # 1. Sync code
+    if no_sync:
+        print(f"[client] skipping sync (--no-sync)", flush=True)
+    else:
+        print(f"[client] syncing v4/ from server...", flush=True)
+        server.push_status(StatusPush(state="syncing", job_id=job.job_id, run_name=job.run_name))
+        _sync_code(server_ssh, server_path, local_path)
+
+    # 2. Launch training subprocess
+    cmd = [sys.executable, "-m", job.module] + job.args + extra_args
+    print(f"[client] running: {' '.join(cmd)}", flush=True)
+    server.push_status(StatusPush(state="running", job_id=job.job_id, run_name=job.run_name))
+
+    log_dir = Path(local_path) / "v4" / "distributed" / "logs"
+    log_dir.mkdir(parents=True, exist_ok=True)
+    log_file = log_dir / f"job_{job.job_id}.log"
+    ctx: dict = {}
+
+    def _tail(path: Path):
+        with open(path, "r") as f:
+            while True:
+                raw = f.readline()
+                if raw:
+                    print(raw, end="", flush=True)
+                    info = _parse_line(raw)
+                    ctx.update(info)
+                    if "epoch" in info:
+                        server.push_status(StatusPush(
+                            state="running",
+                            job_id=job.job_id,
+                            run_name=job.run_name,
+                            fold=ctx.get("fold"),
+                            stage=ctx.get("stage"),
+                            epoch=ctx.get("epoch"),
+                            total_epochs=ctx.get("total_epochs"),
+                            last_val_auc=ctx.get("last_val_auc"),
+                        ))
+                elif proc.poll() is not None:
+                    for raw in f:
+                        print(raw, end="", flush=True)
+                    break
+                else:
+                    time.sleep(0.05)
+
+    with open(log_file, "w") as logf:
+        proc = subprocess.Popen(
+            cmd,
+            stdout=logf,
+            stderr=logf,
+            cwd=local_path,
+            start_new_session=True,
+        )
+
+    tailer = threading.Thread(target=_tail, args=(log_file,), daemon=True)
+    tailer.start()
+    proc.wait()
+    tailer.join(timeout=5)
+    log_file.unlink(missing_ok=True)
+
+    success = proc.returncode == 0
+
+    if not success:
+        try:
+            os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+        except ProcessLookupError:
+            pass
+
+    if not success:
+        print(f"[client] job FAILED (rc={proc.returncode})", flush=True)
+        return False
+
+    # 3. Upload results
+    if no_sync:
+        print(f"[client] skipping upload (--no-sync, results already local)", flush=True)
+    else:
+        print(f"[client] uploading results...", flush=True)
+        server.push_status(StatusPush(state="uploading", job_id=job.job_id, run_name=job.run_name))
+        _upload_results(server_ssh, server_path, local_path, job.run_name, job.output_dir)
+        _clean_local(local_path, job.run_name, job.output_dir)
+
+    print(f"[client] job {job.job_id} complete.", flush=True)
+    return True
+
+
+# ──────────────────────────────────────────────────────────────
+# Compatibility test
+# ──────────────────────────────────────────────────────────────
+
+def _compat_test(server: _Server, server_ssh: str, server_path: str,
+                 local_path: str, config: str, extra_args: list[str],
+                 no_sync: bool = False):
+    """Run 1 fold to verify the env works end-to-end."""
+    print("[client] === compatibility test ===", flush=True)
+    job = JobSpec(
+        job_id="compat-test",
+        run_name="_compat_test",
+        module="v4.classes.v4_hypertower",
+        args=["--config", config, "--device", "cpu"] + extra_args,
+        output_dir="v4/results",
+    )
+    ok = _run_job(job, server, server_ssh, server_path, local_path,
+                  no_sync=no_sync)
+    _clean_local(local_path, "_compat_test", "v4/results")
+    if ok:
+        print("[client] compatibility test PASSED ✓", flush=True)
+    else:
+        print("[client] compatibility test FAILED ✗", flush=True)
+    return ok
+
+
+# ──────────────────────────────────────────────────────────────
+# Main daemon
+# ──────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    ap.add_argument("--server", required=True,
+                    help="Server URL, e.g. http://apollo:8765")
+    ap.add_argument("--token", default=os.environ.get("HT_TOKEN", ""),
+                    help="Shared secret (or set HT_TOKEN env var)")
+    ap.add_argument("--server-ssh", required=True,
+                    help="SSH target for rsync, e.g. rpotter@apollo")
+    ap.add_argument("--server-path", required=True,
+                    help="Absolute path to hypertower root on server")
+    ap.add_argument("--local-path",
+                    default=str(Path.home() / "hypertower"),
+                    help="Absolute path to hypertower root on this machine")
+    ap.add_argument("--poll-interval", type=int, default=15,
+                    help="Seconds to wait between polls when idle")
+    ap.add_argument("--extra-args", nargs=argparse.REMAINDER, default=[],
+                    help="Extra args appended to every job on this client. "
+                         "Use -- to separate: --extra-args -- --device cpu")
+    ap.add_argument("--no-sync", action="store_true",
+                    help="Skip rsync of v4/ before each job (use when client IS the server)")
+    ap.add_argument("--test", action="store_true",
+                    help="Run 1-fold compatibility test and exit")
+    ap.add_argument("--config", default="v4/configs/ensemble_fused.json",
+                    help="Config path for --test mode")
+    args = ap.parse_args()
+
+    if not args.token:
+        ap.error("--token is required (or set HT_TOKEN)")
+
+    hostname = socket.gethostname()
+    gpu_info = _gpu_info()
+    server = _Server(args.server, args.token)
+
+    client_id = server.register(hostname, gpu_info)
+    print(f"[client] registered as {client_id} ({hostname} | {gpu_info})", flush=True)
+
+    if args.test:
+        extra = [a for a in args.extra_args if a != "--"]
+        sys.exit(0 if _compat_test(
+            server, args.server_ssh, args.server_path,
+            args.local_path, args.config, extra,
+            no_sync=args.no_sync,
+        ) else 1)
+
+    print(f"[client] polling every {args.poll_interval}s...", flush=True)
+    while True:
+        try:
+            job = server.poll()
+            if job is None:
+                server.push_status(StatusPush(state="idle"))
+                time.sleep(args.poll_interval)
+                continue
+
+            success = _run_job(
+                job, server,
+                args.server_ssh, args.server_path, args.local_path,
+                no_sync=args.no_sync,
+                extra_args=[a for a in args.extra_args if a != "--"],
+            )
+            server.complete(job.job_id, success,
+                            error_msg=None if success else "non-zero exit code")
+            server.push_status(StatusPush(state="idle"))
+
+        except KeyboardInterrupt:
+            print("\n[client] shutting down", flush=True)
+            break
+        except Exception as exc:
+            print(f"[client] error: {exc}", flush=True)
+            time.sleep(args.poll_interval)
+
+
+if __name__ == "__main__":
+    main()

v4/distributed/hypertower-server.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=HyperTower distributed job server
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+WorkingDirectory=/home/rpotter/hypertower
+Environment=HT_TOKEN=hypertower
+ExecStart=/home/rpotter/miniconda3/envs/fundus_imaging/bin/python -m v4.distributed.server --host 0.0.0.0 --port 8765
+Restart=on-failure
+RestartSec=5
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=default.target

v4/distributed/protocol.py

@@ -0,0 +1,61 @@
+"""Shared data models for server/client communication."""
+from __future__ import annotations
+
+from typing import Optional
+from pydantic import BaseModel
+
+
+class RegisterRequest(BaseModel):
+    hostname: str
+    gpu_info: str
+
+
+class RegisterResponse(BaseModel):
+    client_id: str
+
+
+class StatusPush(BaseModel):
+    state: str                          # idle | syncing | running | uploading | error
+    job_id: Optional[str] = None
+    run_name: Optional[str] = None
+    fold: Optional[int] = None
+    stage: Optional[str] = None
+    epoch: Optional[int] = None
+    total_epochs: Optional[int] = None
+    last_val_auc: Optional[float] = None
+    error: Optional[str] = None
+
+
+class ClientInfo(BaseModel):
+    client_id: str
+    hostname: str
+    gpu_info: str
+    status: StatusPush
+    last_seen: str
+
+
+class JobSpec(BaseModel):
+    job_id: str
+    run_name: str
+    module: str                         # e.g. "v4.classes.v4_hypertower"
+    args: list[str]
+    output_dir: str = "v4/results"
+
+
+class PollResponse(BaseModel):
+    job: Optional[JobSpec] = None
+    please_reregister: bool = False
+
+
+class JobResult(BaseModel):
+    job_id: str
+    success: bool
+    error_msg: Optional[str] = None
+
+
+class JobSubmit(BaseModel):
+    run_name: str
+    module: str = "v4.classes.v4_hypertower"
+    args: list[str]
+    output_dir: str = "v4/results"
+    priority: int = 0

v4/distributed/server.py

@@ -0,0 +1,476 @@
+"""
+HyperTower distributed job server.
+
+Manages a SQLite job queue and a registry of connected clients.
+Clients poll for work, push status updates, and report completion.
+
+Usage:
+    python -m v4.distributed.server --port 8765 --token <secret>
+
+Environment:
+    HT_TOKEN  — fallback if --token is not passed
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sqlite3
+import threading
+import time
+import uuid
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from fastapi import Depends, FastAPI, Header, HTTPException
+import uvicorn
+
+from .protocol import (
+    ClientInfo,
+    JobResult,
+    JobSpec,
+    JobSubmit,
+    PollResponse,
+    RegisterRequest,
+    RegisterResponse,
+    StatusPush,
+)
+
+# ──────────────────────────────────────────────────────────────
+# Global state
+# ──────────────────────────────────────────────────────────────
+
+_TOKEN: str = ""
+_DB_PATH: Path = Path("v4/distributed/jobs.db")
+_REPO_ROOT: Path = Path.cwd()
+_CLIENT_TTL: int = 120  # seconds before a client is considered gone
+_MAX_ATTEMPTS: int = 3  # max times a job is retried before being left as failed
+
+_clients: dict[str, ClientInfo] = {}
+_clients_lock = threading.Lock()
+
+
+def _reap_stale_clients():
+    """Background thread: remove silent clients and re-queue their running jobs."""
+    while True:
+        time.sleep(30)
+        cutoff = datetime.now(timezone.utc).timestamp() - _CLIENT_TTL
+
+        with _clients_lock:
+            stale = [
+                cid
+                for cid, c in _clients.items()
+                if datetime.fromisoformat(c.last_seen).timestamp() < cutoff
+            ]
+            for cid in stale:
+                print(
+                    f"[server] reaped stale client {cid} ({_clients[cid].hostname})",
+                    flush=True,
+                )
+                del _clients[cid]
+            known_ids = set(_clients.keys())
+
+        with _db() as conn:
+            rows = conn.execute(
+                "SELECT job_id, assigned_to FROM jobs WHERE state='running'"
+            ).fetchall()
+            for row in rows:
+                if row["assigned_to"] not in known_ids:
+                    conn.execute(
+                        "UPDATE jobs SET state='pending', assigned_to=NULL, started_at=NULL "
+                        "WHERE job_id=?",
+                        (row["job_id"],),
+                    )
+                    print(
+                        f"[server] re-queued job {row['job_id']} "
+                        f"(client {row['assigned_to']} unknown)",
+                        flush=True,
+                    )
+
+
+# ──────────────────────────────────────────────────────────────
+# Database helpers
+# ──────────────────────────────────────────────────────────────
+
+
+@contextmanager
+def _db():
+    conn = sqlite3.connect(str(_DB_PATH))
+    conn.row_factory = sqlite3.Row
+    try:
+        yield conn
+        conn.commit()
+    finally:
+        conn.close()
+
+
+def _init_db():
+    _DB_PATH.parent.mkdir(parents=True, exist_ok=True)
+    with _db() as conn:
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS jobs (
+                job_id       TEXT PRIMARY KEY,
+                run_name     TEXT NOT NULL,
+                module       TEXT NOT NULL,
+                args         TEXT NOT NULL,   -- JSON list
+                output_dir   TEXT NOT NULL DEFAULT 'v4/results',
+                state        TEXT NOT NULL DEFAULT 'pending',
+                priority     INTEGER NOT NULL DEFAULT 0,
+                assigned_to  TEXT,
+                created_at   TEXT NOT NULL,
+                started_at   TEXT,
+                completed_at TEXT,
+                error_msg    TEXT,
+                attempts     INTEGER NOT NULL DEFAULT 0
+            )
+        """
+        )
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_jobs_state ON jobs(state)")
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_jobs_priority ON jobs(priority)")
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_jobs_run_name ON jobs(run_name)")
+        conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_jobs_args ON jobs(args)")
+        try:
+            conn.execute(
+                "ALTER TABLE jobs ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0"
+            )
+        except Exception:
+            pass  # column already exists
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _ensure_client(client_id: str, hostname: str = "", gpu_info: str = "") -> bool:
+    """Re-register a client that survived a server restart.
+    Returns True if the client was unknown (placeholder created)."""
+    if client_id not in _clients:
+        _clients[client_id] = ClientInfo(
+            client_id=client_id,
+            hostname=hostname or client_id,
+            gpu_info=gpu_info or "unknown",
+            status=StatusPush(state="idle"),
+            last_seen=_now(),
+        )
+        print(f"[server] re-registered {client_id} (survived restart)", flush=True)
+        return True
+    return False
+
+
+# ──────────────────────────────────────────────────────────────
+# FastAPI app
+# ──────────────────────────────────────────────────────────────
+
+app = FastAPI(title="HyperTower Job Server")
+
+
+def _check_token(x_token: str = Header(...)):
+    if x_token != _TOKEN:
+        raise HTTPException(status_code=403, detail="Invalid token")
+
+
+# ── Registration ──────────────────────────────────────────────
+
+
+@app.post(
+    "/register", response_model=RegisterResponse, dependencies=[Depends(_check_token)]
+)
+def register(req: RegisterRequest, reuse_id: Optional[str] = None):
+    with _clients_lock:
+        client_id = (
+            reuse_id if (reuse_id and reuse_id in _clients) else str(uuid.uuid4())[:8]
+        )
+        existing_status = (
+            _clients[client_id].status
+            if client_id in _clients
+            else StatusPush(state="idle")
+        )
+        _clients[client_id] = ClientInfo(
+            client_id=client_id,
+            hostname=req.hostname,
+            gpu_info=req.gpu_info,
+            status=existing_status,
+            last_seen=_now(),
+        )
+    action = "re-registered" if reuse_id else "registered"
+    print(
+        f"[server] {action} {client_id} ({req.hostname} | {req.gpu_info})", flush=True
+    )
+    return RegisterResponse(client_id=client_id)
+
+
+# ── Job polling ───────────────────────────────────────────────
+
+
+@app.post("/poll", response_model=PollResponse, dependencies=[Depends(_check_token)])
+def poll(client_id: str):
+    with _clients_lock:
+        needs_reregister = _ensure_client(client_id)
+        _clients[client_id].last_seen = _now()
+
+    with _db() as conn:
+        row = conn.execute(
+            "SELECT * FROM jobs WHERE state='pending' "
+            "ORDER BY priority DESC, created_at ASC LIMIT 1"
+        ).fetchone()
+
+        if row is None:
+            return PollResponse(job=None, please_reregister=needs_reregister)
+
+        job_id = row["job_id"]
+        cur = conn.execute(
+            "UPDATE jobs SET state='pending', assigned_to=NULL, started_at=NULL "
+            "WHERE assigned_to=? AND state='running' AND job_id!=?",
+            (client_id, job_id),
+        )
+        if cur.rowcount:
+            print(
+                f"[server] reset {cur.rowcount} orphaned running job(s) for {client_id}",
+                flush=True,
+            )
+        conn.execute(
+            "UPDATE jobs SET state='running', assigned_to=?, started_at=? WHERE job_id=?",
+            (client_id, _now(), job_id),
+        )
+
+    job = JobSpec(
+        job_id=job_id,
+        run_name=row["run_name"],
+        module=row["module"],
+        args=json.loads(row["args"]),
+        output_dir=row["output_dir"],
+    )
+
+    with _clients_lock:
+        _clients[client_id].status = StatusPush(
+            state="syncing", job_id=job_id, run_name=row["run_name"]
+        )
+
+    print(f"[server] dispatched {job_id} ({row['run_name']}) → {client_id}", flush=True)
+    return PollResponse(job=job, please_reregister=needs_reregister)
+
+
+# ── Status ────────────────────────────────────────────────────
+
+
+@app.post("/status/{client_id}", dependencies=[Depends(_check_token)])
+def push_status(client_id: str, status: StatusPush):
+    with _clients_lock:
+        needs_reregister = _ensure_client(client_id)
+        _clients[client_id].status = status
+        _clients[client_id].last_seen = _now()
+    return {"ok": True, "please_reregister": needs_reregister}
+
+
+@app.get("/clients", dependencies=[Depends(_check_token)])
+def list_clients():
+    with _clients_lock:
+        return list(_clients.values())
+
+
+@app.get("/clients/{client_id}", dependencies=[Depends(_check_token)])
+def get_client(client_id: str):
+    with _clients_lock:
+        if client_id not in _clients:
+            raise HTTPException(status_code=404, detail="Unknown client")
+        return _clients[client_id]
+
+
+# ── Job completion ────────────────────────────────────────────
+
+
+@app.post("/complete", dependencies=[Depends(_check_token)])
+def complete(result: JobResult):
+    with _db() as conn:
+        if result.success:
+            conn.execute(
+                "UPDATE jobs SET state='done', completed_at=?, error_msg=NULL WHERE job_id=?",
+                (_now(), result.job_id),
+            )
+            print(f"[server] job {result.job_id} → done", flush=True)
+
+            run_row = conn.execute(
+                "SELECT run_name FROM jobs WHERE job_id=?", (result.job_id,)
+            ).fetchone()
+            if run_row:
+                run_name = run_row["run_name"]
+                remaining = conn.execute(
+                    "SELECT COUNT(*) FROM jobs WHERE run_name=? AND state != 'done'",
+                    (run_name,),
+                ).fetchone()[0]
+                if remaining == 0:
+                    total = conn.execute(
+                        "SELECT COUNT(*) FROM jobs WHERE run_name=?", (run_name,)
+                    ).fetchone()[0]
+                    conn.execute("DELETE FROM jobs WHERE run_name=?", (run_name,))
+                    print(
+                        f"[server] run '{run_name}' complete ({total} jobs) — cleared",
+                        flush=True,
+                    )
+        else:
+            row = conn.execute(
+                "SELECT attempts FROM jobs WHERE job_id=?", (result.job_id,)
+            ).fetchone()
+            attempts = (row["attempts"] if row else 0) + 1
+            if attempts < _MAX_ATTEMPTS:
+                conn.execute(
+                    "UPDATE jobs SET state='pending', assigned_to=NULL, started_at=NULL, "
+                    "attempts=?, error_msg=? WHERE job_id=?",
+                    (attempts, result.error_msg, result.job_id),
+                )
+                print(
+                    f"[server] job {result.job_id} failed (attempt {attempts}/{_MAX_ATTEMPTS}), "
+                    f"re-queuing",
+                    flush=True,
+                )
+            else:
+                conn.execute(
+                    "UPDATE jobs SET state='failed', completed_at=?, attempts=?, error_msg=? "
+                    "WHERE job_id=?",
+                    (_now(), attempts, result.error_msg, result.job_id),
+                )
+                print(
+                    f"[server] job {result.job_id} failed permanently after "
+                    f"{attempts} attempts",
+                    flush=True,
+                )
+    return {"ok": True}
+
+
+# ── Job queue management ──────────────────────────────────────
+
+
+@app.post("/jobs", dependencies=[Depends(_check_token)])
+def submit_job(job: JobSubmit):
+    result_dir = _REPO_ROOT / job.output_dir / job.run_name
+    if result_dir.exists() and any(result_dir.rglob("summary.json")):
+        print(f"[server] skipped {job.run_name} (results exist on disk)", flush=True)
+        return {"job_id": "", "duplicate": False, "skipped": True}
+
+    args_json = json.dumps(job.args)
+    job_id    = str(uuid.uuid4())[:12]
+    with _db() as conn:
+        cur = conn.execute(
+            "INSERT OR IGNORE INTO jobs "
+            "(job_id, run_name, module, args, output_dir, priority, created_at) "
+            "VALUES (?,?,?,?,?,?,?)",
+            (job_id, job.run_name, job.module, args_json,
+             job.output_dir, job.priority, _now()),
+        )
+        if cur.rowcount == 0:
+            existing = conn.execute(
+                "SELECT job_id FROM jobs WHERE args=?", (args_json,)
+            ).fetchone()
+            job_id = existing["job_id"]
+            print(f"[server] duplicate ignored ({job.run_name}) → {job_id}", flush=True)
+            return {"job_id": job_id, "duplicate": True}
+    print(f"[server] queued {job_id} ({job.run_name})", flush=True)
+    return {"job_id": job_id, "duplicate": False}
+
+
+@app.get("/jobs", dependencies=[Depends(_check_token)])
+def list_jobs(state: Optional[str] = None):
+    with _db() as conn:
+        if state:
+            rows = conn.execute(
+                "SELECT * FROM jobs WHERE state=? ORDER BY created_at DESC", (state,)
+            ).fetchall()
+        else:
+            rows = conn.execute(
+                "SELECT * FROM jobs ORDER BY created_at DESC"
+            ).fetchall()
+    return [dict(r) for r in rows]
+
+
+@app.post("/jobs/clear", dependencies=[Depends(_check_token)])
+def clear_jobs(body: dict):
+    with _db() as conn:
+        if body.get("all"):
+            cur = conn.execute("DELETE FROM jobs")
+        elif body.get("run_name"):
+            cur = conn.execute("DELETE FROM jobs WHERE run_name=?", (body["run_name"],))
+        else:
+            states = body.get("states", ["done", "failed", "cancelled"])
+            placeholders = ",".join("?" * len(states))
+            cur = conn.execute(
+                f"DELETE FROM jobs WHERE state IN ({placeholders})", states
+            )
+    print(f"[server] cleared {cur.rowcount} jobs", flush=True)
+    return {"cleared": cur.rowcount}
+
+
+@app.delete("/jobs/{job_id}", dependencies=[Depends(_check_token)])
+def cancel_job(job_id: str):
+    with _db() as conn:
+        conn.execute(
+            "UPDATE jobs SET state='cancelled' WHERE job_id=? AND state='pending'",
+            (job_id,),
+        )
+    return {"ok": True}
+
+
+# ──────────────────────────────────────────────────────────────
+# Entry point
+# ──────────────────────────────────────────────────────────────
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    ap.add_argument("--port", type=int, default=8765)
+    ap.add_argument("--host", default="0.0.0.0")
+    ap.add_argument(
+        "--token",
+        default=os.environ.get("HT_TOKEN", ""),
+        help="Shared secret (or set HT_TOKEN env var)",
+    )
+    ap.add_argument(
+        "--db", default="v4/distributed/jobs.db", help="Path to SQLite job database"
+    )
+    ap.add_argument(
+        "--client-ttl",
+        type=int,
+        default=120,
+        help="Seconds of silence before a client is reaped (default: 120)",
+    )
+    ap.add_argument(
+        "--max-attempts",
+        type=int,
+        default=3,
+        help="Max times a failed job is retried before being left as failed (default: 3)",
+    )
+    ap.add_argument(
+        "--root",
+        default="",
+        help="Repo root for results-existence checks (default: cwd)",
+    )
+    args = ap.parse_args()
+
+    if not args.token:
+        ap.error("--token is required (or set HT_TOKEN)")
+
+    global _TOKEN, _DB_PATH, _REPO_ROOT, _CLIENT_TTL, _MAX_ATTEMPTS
+    _TOKEN = args.token
+    _DB_PATH = Path(args.db)
+    _REPO_ROOT = Path(args.root).resolve() if args.root else Path.cwd()
+    _CLIENT_TTL = args.client_ttl
+    _MAX_ATTEMPTS = args.max_attempts
+    _init_db()
+
+    reaper = threading.Thread(target=_reap_stale_clients, daemon=True)
+    reaper.start()
+
+    print(
+        f"[server] listening on {args.host}:{args.port}  client_ttl={_CLIENT_TTL}s",
+        flush=True,
+    )
+    uvicorn.run(app, host=args.host, port=args.port, log_level="warning")
+
+
+if __name__ == "__main__":
+    main()

v4/distributed/server_cheat_sheet.md

@@ -0,0 +1,147 @@
+# Distributed Server Cheat Sheet
+
+All commands assume server is running on hades at port 8765.
+
+## Start Server
+```bash
+python -m v4.distributed.server --token hypertower
+```
+Run inside tmux so it survives disconnects:
+```bash
+tmux new -s htserver
+python -m v4.distributed.server --token hypertower
+# Ctrl-B D to detach
+tmux attach -t htserver   # reattach later
+```
+
+## Local Workflow (hades as server + client)
+
+```bash
+# Terminal 1 — server
+python -m v4.distributed.server --token hypertower
+
+# Terminal 2 — client (skip rsync, results already local)
+python -m v4.distributed.client \
+    --server http://hades:8765 --token hypertower \
+    --server-ssh ignored --server-path ignored \
+    --local-path /home/rpotter/hypertower \
+    --no-sync
+
+# Terminal 3 — dispatch (exits after queuing; client picks up jobs)
+python -m v4.distributed.batch_dispatch \
+    --server http://hades:8765 --token hypertower \
+    --config v4/configs/ensemble_fused.json \
+    --batch v4/scripts/experiments/my_batch.json
+```
+
+---
+
+## Start Clients
+
+**Hades (server-local, no sync):**
+```bash
+python -m v4.distributed.client \
+    --server http://hades:8765 --token hypertower \
+    --server-ssh ignored --server-path ignored \
+    --local-path /home/rpotter/hypertower \
+    --no-sync
+```
+
+**Apollo (remote client):**
+```bash
+python -m v4.distributed.client \
+    --server http://hades:8765 --token hypertower \
+    --server-ssh rpotter@hades \
+    --server-path /home/rpotter/hypertower \
+    --local-path /home/odin/hypertower
+```
+
+---
+
+## Monitoring
+
+**Live client monitor (refreshes every 5s):**
+```bash
+python -m v4.distributed.cli --server http://hades:8765 --token hypertower clients --watch
+```
+
+**Faster refresh:**
+```bash
+python -m v4.distributed.cli --server http://hades:8765 --token hypertower clients --watch --interval 2
+```
+
+**Inspect a single client:**
+```bash
+python -m v4.distributed.cli --server http://hades:8765 --token hypertower clients <client_id>
+```
+
+**View job queue:**
+```bash
+python -m v4.distributed.cli --server http://hades:8765 --token hypertower jobs
+```
+
+**Filter by state:**
+```bash
+python -m v4.distributed.cli --server http://hades:8765 --token hypertower jobs --state pending
+python -m v4.distributed.cli --server http://hades:8765 --token hypertower jobs --state running
+python -m v4.distributed.cli --server http://hades:8765 --token hypertower jobs --state failed
+```
+
+---
+
+## Submitting Jobs
+
+**Batch dispatch (dry run first):**
+```bash
+python -m v4.distributed.batch_dispatch \
+    --server http://hades:8765 --token hypertower \
+    --config v4/configs/ensemble_fused.json \
+    --batch v4/scripts/experiments/fusion_dim_sweep.json \
+    --dry-run
+```
+
+**Live submit:**
+```bash
+python -m v4.distributed.batch_dispatch \
+    --server http://hades:8765 --token hypertower \
+    --config v4/configs/ensemble_fused.json \
+    --batch v4/scripts/experiments/fusion_dim_sweep.json
+```
+
+**Fewer reps (e.g. quick test):**
+```bash
+python -m v4.distributed.batch_dispatch \
+    --server http://hades:8765 --token hypertower \
+    --config v4/configs/ensemble_fused.json \
+    --batch v4/scripts/experiments/fusion_dim_sweep.json \
+    --reps 3
+```
+
+---
+
+## Queue Management
+
+**Clear failed jobs:**
+```bash
+python -m v4.distributed.cli --server http://hades:8765 --token hypertower clear --states failed
+```
+
+**Clear running jobs (orphan cleanup):**
+```bash
+python -m v4.distributed.cli --server http://hades:8765 --token hypertower clear --states running
+```
+
+**Clear all jobs:**
+```bash
+python -m v4.distributed.cli --server http://hades:8765 --token hypertower clear --all
+```
+
+**Clear by run name:**
+```bash
+python -m v4.distributed.cli --server http://hades:8765 --token hypertower clear --run-name experiments/fusion_dim_sweep/dim256
+```
+
+**Cancel a specific job:**
+```bash
+python -m v4.distributed.cli --server http://hades:8765 --token hypertower cancel <job_id>
+```