hypertower/scripts/grid_search_analytics/rerun_grid_item.py

#!/usr/bin/env python3
"""Re-run a single grid-search configuration into analysis_data/re_runs."""
from __future__ import annotations

import argparse
import csv
import shutil
import subprocess
import sys
from pathlib import Path
from typing import Dict, List


# ---------------------------
# Config (edit in IDE)
# ---------------------------
RUN_ID = "20251129-0063"  # fallback if --run-number is not provided
OUTPUT_RUN_ID = RUN_ID  # fallback output run id
SHORTNAME = "re_runs"  # output root under analysis_data/ and models/
GRID_PLAN = Path("analysis_data/grid_search/grid_plan.csv")
MANIFEST = Path("manifest.csv")
RUN_SCRIPT = Path("scripts/run_multifold.py")
REBUILD_SCRIPT = Path("scripts/rebuild_run_best_plots.py")
PLOT_HEADS = ["fused", "image", "metadata"]
USE_HOLDOUT_BEST_FOR_PLOTS = True
OVERWRITE_HOLDOUT_PROBS = True
ALLOW_EXISTING_RUN_DIR = False


def _parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description="Re-run a single grid-search item.")
    ap.add_argument(
        "--run-number",
        type=str,
        default=None,
        help="Last 4 digits of run_id (e.g., 0063).",
    )
    ap.add_argument(
        "--output-run-id",
        type=str,
        default=None,
        help="Optional output run id; defaults to matched run_id.",
    )
    return ap.parse_args()


def _read_plan(path: Path) -> List[Dict[str, str]]:
    if not path.exists():
        raise FileNotFoundError(f"Grid plan not found: {path}")
    with path.open(newline="") as fh:
        reader = csv.DictReader(fh)
        return list(reader)


def _find_row(rows: List[Dict[str, str]], run_id: str) -> Dict[str, str]:
    for row in rows:
        if row.get("run_id") == run_id:
            return row
    raise ValueError(f"run_id not found in grid plan: {run_id}")


def _resolve_run_id(rows: List[Dict[str, str]], run_number: str | None) -> str:
    if not run_number:
        return RUN_ID
    run_number = str(run_number).strip()
    if run_number.isdigit():
        run_number = run_number.zfill(4)
    matches = [
        r.get("run_id", "")
        for r in rows
        if str(r.get("run_id", "")).endswith(f"-{run_number}")
    ]
    if len(matches) == 1:
        return matches[0]
    if len(matches) > 1:
        raise ValueError(
            f"Multiple run_ids matched run-number '{run_number}': {matches[:5]}{' ...' if len(matches) > 5 else ''}"
        )
    raise ValueError(f"No run_id found ending with '-{run_number}'")


def _build_run_command(row: Dict[str, str], output_run_id: str) -> List[str]:
    cmd = [
        sys.executable,
        str(RUN_SCRIPT),
        "--backbone",
        "resnet50",
        "--fusion-mode",
        "fused",
        "--epochs",
        "40",
        "--batch-size",
        "8",
        "--img-crop-manifest",
        str(MANIFEST),
        "--img-crop-weights",
        row["crop_weights"],
        "--img-crop-normalize",
        row["crop_normalize"],
        "--eval_mode",
        row["eval_mode"],
        "--holdout-per-class",
        "12",
        "--run-id",
        output_run_id,
        "--shortname",
        SHORTNAME,
    ]
    if row.get("crop_tta") == "True":
        cmd.append("--img-crop-tta")

    loss_mode = row.get("loss_mode")
    if loss_mode == "focal":
        cmd.extend(["--focal-gamma", "2.0"])
    elif loss_mode == "balanced":
        cmd.append("--balanced-sampler")

    thaw_mode = row.get("thaw_mode")
    if thaw_mode == "gradual":
        cmd.append("--gradual-thaw")
        cmd.extend(["--thaw-ratio", "0.33"])
        cmd.extend(["--thaw-start-epoch", "10"])
        cmd.extend(["--thaw-target", "image"])

    se_mode = row.get("se_mode")
    if se_mode == "none":
        cmd.append("--no-se")
    else:
        cmd.extend(["--se-reduction", "16"])
        cmd.extend(["--se-reduction-tower", "16"])
        cmd.extend(["--se-where", se_mode])
        bridge_pre = row.get("se_bridge_pre_norm")
        tower_pre = row.get("se_tower_pre_norm")
        if bridge_pre == "True":
            cmd.append("--se-pre-norm")
        elif bridge_pre == "False":
            cmd.append("--no-se-pre-norm")
        if tower_pre == "True":
            cmd.append("--se-pre-norm-tower")
        elif tower_pre == "False":
            cmd.append("--no-se-pre-norm-tower")

    return cmd


def _swap_in_holdout_best(models_dir: Path) -> None:
    for fold_dir in sorted(models_dir.glob("fold*")):
        if not fold_dir.is_dir():
            continue
        holdout_best = fold_dir / "model_holdout_best.pt"
        model_best = fold_dir / "model_best.pt"
        if not holdout_best.exists():
            print(f"[warn] {holdout_best} missing; skipping.")
            continue
        if model_best.exists():
            backup = fold_dir / "model_best_from_train.pt"
            if not backup.exists():
                try:
                    shutil.copy2(model_best, backup)
                except Exception:
                    pass
        try:
            shutil.copy2(holdout_best, model_best)
        except Exception as exc:
            print(f"[warn] failed to replace {model_best}: {exc}")


def _run_rebuild(run_dir: Path) -> None:
    for head in PLOT_HEADS:
        cmd = [
            sys.executable,
            str(REBUILD_SCRIPT),
            "--run-dir",
            str(run_dir),
            "--head",
            head,
            "--use-holdout",
        ]
        if OVERWRITE_HOLDOUT_PROBS:
            cmd.append("--overwrite")
        print("[rerun] Rebuilding holdout ROC plots:", " ".join(cmd))
        subprocess.run(cmd, check=True)


def main() -> None:
    args = _parse_args()
    rows = _read_plan(GRID_PLAN)
    run_id = _resolve_run_id(rows, args.run_number)
    row = _find_row(rows, run_id)
    output_run_id = args.output_run_id or run_id

    run_dir = Path("analysis_data") / SHORTNAME / output_run_id
    if run_dir.exists() and not ALLOW_EXISTING_RUN_DIR:
        raise SystemExit(
            f"Run directory already exists: {run_dir} (set ALLOW_EXISTING_RUN_DIR=True to reuse)"
        )

    cmd = _build_run_command(row, output_run_id=output_run_id)
    print("[rerun] Launching:", " ".join(cmd))
    subprocess.run(cmd, check=True)

    models_dir = Path("models") / SHORTNAME / output_run_id
    if USE_HOLDOUT_BEST_FOR_PLOTS:
        print("[rerun] Swapping in holdout-best checkpoints for plotting.")
        _swap_in_holdout_best(models_dir)

    _run_rebuild(run_dir)
    print(f"[rerun] Done. Outputs in {run_dir}")


if __name__ == "__main__":
    main()