moved_repo_first_update
This commit is contained in:
rpotter6298
Executable
+431
@@ -0,0 +1,431 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Grid-search runner for run_multifold experiments.
|
||||
|
||||
Features:
|
||||
* Enumerates the requested configuration grid and writes grid_plan.csv.
|
||||
* Picks the next incomplete run, marks it running, executes run_multifold.py.
|
||||
* Records AUC/accuracy metrics per fold into grid_report.csv.
|
||||
* Removes model checkpoints for runs dominated (80%+ metrics worse) by others.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import fcntl
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
RUN_SCRIPT = REPO_ROOT / "scripts" / "run_multifold.py"
|
||||
MANIFEST = REPO_ROOT / "manifest.csv"
|
||||
GRID_DIR = REPO_ROOT / "analysis_data" / "grid_search"
|
||||
PLAN_PATH = GRID_DIR / "grid_plan.csv"
|
||||
REPORT_PATH = GRID_DIR / "grid_report.csv"
|
||||
LOCK_PATH = GRID_DIR / ".grid_lock"
|
||||
MODELS_ROOT = REPO_ROOT / "models" / "grid_search"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
ap = argparse.ArgumentParser(description="Grid-search orchestrator for run_multifold.")
|
||||
ap.add_argument("--plan-date", default=datetime.now().strftime("%Y%m%d"),
|
||||
help="Date prefix used when generating run IDs (default: today).")
|
||||
ap.add_argument("--regen-plan", action="store_true",
|
||||
help="Rebuild the grid plan from scratch (overwrites existing plan).")
|
||||
ap.add_argument("--manifest", type=Path, default=MANIFEST,
|
||||
help="UNet manifest CSV for cropper.")
|
||||
ap.add_argument("--weights-dir", type=Path, default=REPO_ROOT / "models" / "unet_segmenter",
|
||||
help="Directory containing norm_* subfolders with best.pt.")
|
||||
ap.add_argument("--dry-run", action="store_true", help="Enumerate next run without executing.")
|
||||
ap.add_argument("--max-runs", type=int, default=1,
|
||||
help="Maximum runs to execute in this invocation (default: 1).")
|
||||
ap.add_argument("--run-all", action="store_true",
|
||||
help="Execute runs sequentially until plan is exhausted (overrides --max-runs).")
|
||||
return ap.parse_args()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def file_lock(lock_path: Path):
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(lock_path, "w") as lock_file:
|
||||
fcntl.flock(lock_file, fcntl.LOCK_EX)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
fcntl.flock(lock_file, fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def read_csv(path: Path) -> List[Dict[str, str]]:
|
||||
if not path.exists():
|
||||
return []
|
||||
with path.open(newline="") as fh:
|
||||
reader = csv.DictReader(fh)
|
||||
return list(reader)
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: List[Dict[str, str]], headers: List[str]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", newline="") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=headers)
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
writer.writerow(row)
|
||||
|
||||
|
||||
def grid_configs(base_date: str, weights_dir: Path) -> List[Dict[str, str]]:
|
||||
eval_modes = ["binary", "multiclass"]
|
||||
crop_variants = [
|
||||
("norm_imagenet", "imagenet"),
|
||||
("normalize_none", "none"),
|
||||
("norm_per_image", "per_image"),
|
||||
]
|
||||
tta_opts = [False, True]
|
||||
loss_modes = ["focal", "balanced", "none"]
|
||||
thaw_modes = ["none", "gradual"]
|
||||
|
||||
se_configs = []
|
||||
# none
|
||||
se_configs.append(("none", {"se_enabled": False}))
|
||||
# bridge only
|
||||
for pre in (True, False):
|
||||
se_configs.append((
|
||||
"bridge",
|
||||
{"se_enabled": True, "se_where": "bridge", "bridge_pre_norm": pre, "tower_pre_norm": None},
|
||||
))
|
||||
# tower only
|
||||
for pre in (True, False):
|
||||
se_configs.append((
|
||||
"tower",
|
||||
{"se_enabled": True, "se_where": "tower", "bridge_pre_norm": None, "tower_pre_norm": pre},
|
||||
))
|
||||
# both (four combos)
|
||||
for b_pre in (True, False):
|
||||
for t_pre in (True, False):
|
||||
se_configs.append((
|
||||
"both",
|
||||
{
|
||||
"se_enabled": True,
|
||||
"se_where": "both",
|
||||
"bridge_pre_norm": b_pre,
|
||||
"tower_pre_norm": t_pre,
|
||||
},
|
||||
))
|
||||
|
||||
combos = []
|
||||
idx = 0
|
||||
for eval_mode in eval_modes:
|
||||
for variant, norm in crop_variants:
|
||||
weights_path = weights_dir / variant / "best.pt"
|
||||
for tta in tta_opts:
|
||||
for loss in loss_modes:
|
||||
for thaw in thaw_modes:
|
||||
for se_name, se_opts in se_configs:
|
||||
run_id = f"{base_date}-{idx:04d}"
|
||||
combos.append({
|
||||
"run_id": run_id,
|
||||
"status": "incomplete",
|
||||
"eval_mode": eval_mode,
|
||||
"crop_variant": variant,
|
||||
"crop_normalize": norm,
|
||||
"crop_weights": str(weights_path),
|
||||
"crop_tta": str(tta),
|
||||
"loss_mode": loss,
|
||||
"thaw_mode": thaw,
|
||||
"se_mode": se_name,
|
||||
"se_bridge_pre_norm": str(se_opts.get("bridge_pre_norm")),
|
||||
"se_tower_pre_norm": str(se_opts.get("tower_pre_norm")),
|
||||
})
|
||||
idx += 1
|
||||
return combos
|
||||
|
||||
|
||||
PLAN_HEADERS = [
|
||||
"run_id",
|
||||
"status",
|
||||
"eval_mode",
|
||||
"crop_variant",
|
||||
"crop_normalize",
|
||||
"crop_weights",
|
||||
"crop_tta",
|
||||
"loss_mode",
|
||||
"thaw_mode",
|
||||
"se_mode",
|
||||
"se_bridge_pre_norm",
|
||||
"se_tower_pre_norm",
|
||||
]
|
||||
|
||||
|
||||
def ensure_plan(args: argparse.Namespace) -> None:
|
||||
if args.regen_plan or not PLAN_PATH.exists():
|
||||
combos = grid_configs(args.plan_date, args.weights_dir)
|
||||
write_csv(PLAN_PATH, combos, PLAN_HEADERS)
|
||||
print(f"[grid] Plan created with {len(combos)} runs at {PLAN_PATH}")
|
||||
|
||||
|
||||
def select_next_run() -> Optional[Dict[str, str]]:
|
||||
rows = read_csv(PLAN_PATH)
|
||||
for row in rows:
|
||||
if row["status"] == "incomplete":
|
||||
row["status"] = "running"
|
||||
write_csv(PLAN_PATH, rows, PLAN_HEADERS)
|
||||
return row
|
||||
return None
|
||||
|
||||
|
||||
def update_run_status(run_id: str, new_status: str) -> None:
|
||||
rows = read_csv(PLAN_PATH)
|
||||
for row in rows:
|
||||
if row["run_id"] == run_id:
|
||||
row["status"] = new_status
|
||||
break
|
||||
write_csv(PLAN_PATH, rows, PLAN_HEADERS)
|
||||
|
||||
|
||||
def build_run_command(row: Dict[str, str], manifest: Path) -> List[str]:
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(RUN_SCRIPT),
|
||||
"--backbone",
|
||||
"resnet50",
|
||||
"--fusion-mode",
|
||||
"fused",
|
||||
"--epochs",
|
||||
"40",
|
||||
"--batch-size",
|
||||
"8",
|
||||
"--img-crop-manifest",
|
||||
str(manifest),
|
||||
"--img-crop-weights",
|
||||
row["crop_weights"],
|
||||
"--img-crop-normalize",
|
||||
row["crop_normalize"],
|
||||
"--eval_mode",
|
||||
row["eval_mode"],
|
||||
"--holdout-per-class",
|
||||
"12",
|
||||
"--run-id",
|
||||
row["run_id"],
|
||||
"--shortname",
|
||||
"grid_search",
|
||||
]
|
||||
if row["crop_tta"] == "True":
|
||||
cmd.append("--img-crop-tta")
|
||||
|
||||
# Loss/balancing modes
|
||||
if row["loss_mode"] == "focal":
|
||||
cmd.extend(["--focal-gamma", "2.0"])
|
||||
elif row["loss_mode"] == "balanced":
|
||||
cmd.append("--balanced-sampler")
|
||||
|
||||
# Thaw schedule
|
||||
if row["thaw_mode"] == "gradual":
|
||||
cmd.append("--gradual-thaw")
|
||||
cmd.extend(["--thaw-ratio", "0.33"])
|
||||
cmd.extend(["--thaw-start-epoch", "10"])
|
||||
cmd.extend(["--thaw-target", "image"])
|
||||
|
||||
# SE settings
|
||||
if row["se_mode"] == "none":
|
||||
cmd.append("--no-se")
|
||||
else:
|
||||
cmd.extend(["--se-reduction", "16"])
|
||||
cmd.extend(["--se-reduction-tower", "16"])
|
||||
cmd.extend(["--se-where", row["se_mode"]])
|
||||
bridge_pre = row["se_bridge_pre_norm"]
|
||||
tower_pre = row["se_tower_pre_norm"]
|
||||
if bridge_pre == "True":
|
||||
cmd.append("--se-pre-norm")
|
||||
elif bridge_pre == "False":
|
||||
cmd.append("--no-se-pre-norm")
|
||||
if tower_pre == "True":
|
||||
cmd.append("--se-pre-norm-tower")
|
||||
elif tower_pre == "False":
|
||||
cmd.append("--no-se-pre-norm-tower")
|
||||
return cmd
|
||||
|
||||
|
||||
def run_command(cmd: List[str]) -> None:
|
||||
print("[grid] Launching:", " ".join(cmd))
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
|
||||
METRIC_KEYS = ["auc_fused", "auc_img", "auc_md", "acc_fused", "acc_img", "acc_md"]
|
||||
|
||||
|
||||
def extract_metrics(run_id: str) -> Dict[str, str]:
|
||||
summary_path = REPO_ROOT / "analysis_data" / "grid_search" / run_id / "summary.json"
|
||||
if not summary_path.exists():
|
||||
raise FileNotFoundError(f"Missing summary.json for run {run_id}")
|
||||
with summary_path.open() as fh:
|
||||
summary = json.load(fh)
|
||||
|
||||
rows = {}
|
||||
for fold in summary.get("fold_metrics", []):
|
||||
if not isinstance(fold, dict):
|
||||
continue
|
||||
f_idx = fold.get("fold")
|
||||
stats = fold.get("stats") or {}
|
||||
if not isinstance(stats, dict):
|
||||
continue
|
||||
for key in METRIC_KEYS:
|
||||
val = stats.get(key)
|
||||
if val is None:
|
||||
continue
|
||||
rows[f"metric_fold{f_idx}_{key}"] = str(val)
|
||||
best_mean = summary.get("best_metric_mean")
|
||||
if best_mean is not None:
|
||||
rows["metric_best_mean"] = str(best_mean)
|
||||
return rows
|
||||
|
||||
|
||||
def update_report(row: Dict[str, str], metrics: Dict[str, str]) -> None:
|
||||
existing = read_csv(REPORT_PATH)
|
||||
# Remove existing entry for run_id
|
||||
existing = [r for r in existing if r.get("run_id") != row["run_id"]]
|
||||
record = {**row, **metrics}
|
||||
existing.append(record)
|
||||
headers = sorted({key for r in existing for key in r.keys()})
|
||||
write_csv(REPORT_PATH, existing, headers)
|
||||
|
||||
|
||||
def load_report_rows() -> List[Dict[str, str]]:
|
||||
return read_csv(REPORT_PATH)
|
||||
|
||||
|
||||
def metric_columns(rows: List[Dict[str, str]]) -> List[str]:
|
||||
keys = set()
|
||||
for row in rows:
|
||||
for key in row:
|
||||
if key.startswith("metric_"):
|
||||
keys.add(key)
|
||||
return sorted(keys)
|
||||
|
||||
|
||||
def _to_float(val: str) -> Optional[float]:
|
||||
try:
|
||||
f = float(val)
|
||||
if math.isnan(f):
|
||||
return None
|
||||
return f
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def prune_dominated(rows: List[Dict[str, str]]) -> None:
|
||||
"""
|
||||
Remove model directories for runs that are clearly dominated by another run.
|
||||
A run is dominated if:
|
||||
* Another run has a strictly higher metric_best_mean, OR
|
||||
* Another run is >= on >=80% of overlapping metrics and strictly better on at least one.
|
||||
"""
|
||||
metrics = metric_columns(rows)
|
||||
if not metrics:
|
||||
return
|
||||
|
||||
dominated = set()
|
||||
for row in rows:
|
||||
run_id = row["run_id"]
|
||||
row_vals = {m: row.get(m) for m in metrics}
|
||||
row_best = _to_float(row_vals.get("metric_best_mean"))
|
||||
|
||||
for other in rows:
|
||||
if other["run_id"] == run_id:
|
||||
continue
|
||||
|
||||
other_vals = {m: other.get(m) for m in metrics}
|
||||
other_best = _to_float(other_vals.get("metric_best_mean"))
|
||||
|
||||
# Fast path: compare aggregate best mean if both have it
|
||||
if row_best is not None and other_best is not None and other_best > row_best:
|
||||
dominated.add(run_id)
|
||||
break
|
||||
|
||||
# Fallback: overlap-wise dominance
|
||||
comparisons = []
|
||||
better = 0
|
||||
for key in metrics:
|
||||
v1 = _to_float(row_vals.get(key))
|
||||
v2 = _to_float(other_vals.get(key))
|
||||
if v1 is None or v2 is None:
|
||||
continue
|
||||
comparisons.append(v2 >= v1)
|
||||
if v2 > v1:
|
||||
better += 1
|
||||
if not comparisons:
|
||||
continue
|
||||
fraction = sum(comparisons) / len(comparisons)
|
||||
if fraction >= 0.8 and better > 0:
|
||||
dominated.add(run_id)
|
||||
break
|
||||
|
||||
for run_id in dominated:
|
||||
model_dir = MODELS_ROOT / run_id
|
||||
if model_dir.exists():
|
||||
print(f"[grid] Removing dominated model artifacts for {run_id}")
|
||||
try:
|
||||
shutil.rmtree(model_dir)
|
||||
except OSError as exc:
|
||||
# Don't fail the grid run if cleanup isn't permitted (e.g., locked SMB dirs).
|
||||
print(f"[grid] Warning: could not remove {model_dir}: {exc}")
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
ensure_plan(args)
|
||||
if args.dry_run:
|
||||
with file_lock(LOCK_PATH):
|
||||
next_run = select_next_run()
|
||||
if next_run is None:
|
||||
print("[grid] No incomplete runs remaining.")
|
||||
return
|
||||
update_run_status(next_run["run_id"], "incomplete")
|
||||
print("[grid] Next run:", next_run)
|
||||
return
|
||||
|
||||
max_runs = None if args.run_all else args.max_runs
|
||||
runs_done = 0
|
||||
|
||||
while True:
|
||||
with file_lock(LOCK_PATH):
|
||||
next_run = select_next_run()
|
||||
if next_run is None:
|
||||
if runs_done == 0:
|
||||
print("[grid] All runs completed.")
|
||||
else:
|
||||
print(f"[grid] No more runs remaining after {runs_done} run(s).")
|
||||
return
|
||||
|
||||
run_id = next_run["run_id"]
|
||||
try:
|
||||
cmd = build_run_command(next_run, args.manifest)
|
||||
run_command(cmd)
|
||||
metrics = extract_metrics(run_id)
|
||||
with file_lock(LOCK_PATH):
|
||||
update_run_status(run_id, "completed")
|
||||
update_report(next_run, metrics)
|
||||
report_rows = load_report_rows()
|
||||
prune_dominated(report_rows)
|
||||
print(f"[grid] Run {run_id} completed.")
|
||||
except Exception as exc:
|
||||
with file_lock(LOCK_PATH):
|
||||
update_run_status(run_id, "incomplete")
|
||||
raise SystemExit(f"[grid] Run {run_id} failed: {exc}") from exc
|
||||
|
||||
runs_done += 1
|
||||
if max_runs is not None and runs_done >= max_runs:
|
||||
print(f"[grid] Reached run limit ({max_runs}); stopping.")
|
||||
return
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user