ryan/hypertower
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

logging rework temp save

Browse Source
This commit is contained in:
rpotter6298
2026-03-04 09:40:09 +01:00
parent 080b5999fa
commit d36b9508ff
41 changed files with 2182 additions and 383 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/output_analysis/visualizations/aggregate_roc_perclass_all_models_v2.py
Executable
+281
View File
@@ -0,0 +1,281 @@
#!/usr/bin/env python3
"""
Per-class ROC curves for v2 HyperTower runs.
Plots multiple runs as separate lines on the same axes — one figure per class
(multiclass) or one figure total (binary).
v2 directory layout
-------------------
analysis_data/{run_name}/{eval_mode}/{tower_mode}/
fold0/ y_true.npy probs_fused.npy | probs_bilat.npy | probs_classic.npy | probs_fused_head.npy
fold1/ ...
Usage examples
--------------
# Compare UNet ensemble vs bilateral vs fused head (binary)
python scripts/output_analysis/visualizations/aggregate_roc_perclass_all_models_v2.py \\
--mode binary --tag unet_binary_comparison \\
--runs \\
analysis_data/v2_modes_full_40ep_5fold_roi_unet_perimage_refugebuild_holdout/binary/ensemble:"UNet Ensemble" \\
analysis_data/v2_modes_full_40ep_5fold_roi_unet_perimage_refugebuild_holdout/binary/bilateral:"UNet Bilateral" \\
analysis_data/v2_ensemble_fused_binary_unet_40ep_5fold_v1/binary/ensemble:"UNet Fused Head"
Each --runs entry is <path>:<label> where <path> points directly to the
{eval_mode}/{tower_mode} subdirectory and <label> is shown in the legend.
Outputs (written to --output-dir, default: analysis_data/roc_plots/)
{tag}_binary_roc.png (binary mode)
{tag}_class{k}_roc.png (multiclass mode, one file per class)
{tag}_roc_summary.json
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, auc as sk_auc
# ---------------------------------------------------------------------------
# Probs filename auto-detection priority per tower mode
# ---------------------------------------------------------------------------
_PROBS_PRIORITY: dict[str, list[str]] = {
"ensemble": ["probs_fused_head", "probs_fused"],
"bilateral": ["probs_bilat"],
"single": ["probs_classic"],
"classic": ["probs_classic"],
}
_ALL_PROBS = ["probs_fused_head", "probs_fused", "probs_bilat", "probs_classic"]
def _detect_probs_stem(fold_dir: Path, tower_mode: str | None) -> str | None:
priority = _PROBS_PRIORITY.get(tower_mode, _ALL_PROBS) if tower_mode else _ALL_PROBS
for stem in priority:
if (fold_dir / f"{stem}.npy").exists():
return stem
return None
# ---------------------------------------------------------------------------
# Fold discovery and loading
# ---------------------------------------------------------------------------
def find_fold_dirs(mode_dir: Path) -> list[Path]:
return sorted(
[p for p in mode_dir.iterdir() if p.is_dir() and p.name.startswith("fold")],
key=lambda p: int(p.name.replace("fold", "")),
)
def load_fold(fold_dir: Path, probs_stem: str) -> tuple[np.ndarray, np.ndarray] | None:
y_path = fold_dir / "y_true.npy"
p_path = fold_dir / f"{probs_stem}.npy"
if not y_path.exists() or not p_path.exists():
return None
return np.load(y_path), np.load(p_path)
# ---------------------------------------------------------------------------
# Per-class ROC helpers
# ---------------------------------------------------------------------------
def per_class_roc(y: np.ndarray, p: np.ndarray) -> dict[int, tuple]:
K = p.shape[1]
out: dict[int, tuple] = {}
for k in range(K):
yb = (y == k).astype(np.uint8)
if yb.sum() == 0 or yb.sum() == len(yb):
continue
fpr, tpr, _ = roc_curve(yb, p[:, k])
out[k] = (fpr, tpr, sk_auc(fpr, tpr))
return out
def build_mean_curve(mode_dir: Path, tower_mode: str | None, mode: str) -> dict | None:
fold_dirs = find_fold_dirs(mode_dir)
if not fold_dirs:
return None
probs_stem: str | None = None
for fd in fold_dirs:
probs_stem = _detect_probs_stem(fd, tower_mode)
if probs_stem:
break
if probs_stem is None:
return None
grid = np.linspace(0, 1, 501)
per_fold: list[dict] = []
for fd in fold_dirs:
result = load_fold(fd, probs_stem)
if result is None:
continue
y, p = result
if mode == "binary":
mask = np.isin(y, [0, 1])
y, p = y[mask], p[mask]
if p.shape[1] > 2:
p = p[:, :2]
per_fold.append(per_class_roc(y, p))
if not per_fold:
return None
K = max(max(d.keys()) for d in per_fold) + 1
class_curves: dict[int, dict] = {}
for k in range(K):
tprs, aucs = [], []
for d in per_fold:
if k not in d:
continue
fpr, tpr, a = d[k]
tprs.append(np.interp(grid, fpr, tpr))
aucs.append(a)
if not tprs:
continue
tprs_arr = np.vstack(tprs)
class_curves[k] = {
"fpr": grid,
"tpr_mean": tprs_arr.mean(axis=0),
"tpr_std": tprs_arr.std(axis=0),
"auc_mean": float(np.nanmean(aucs)),
"auc_std": float(np.nanstd(aucs)),
}
return {"probs_stem": probs_stem, "class_curves": class_curves}
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def parse_run_entry(entry: str) -> tuple[Path, str]:
"""Parse path:label or path (label defaults to last two dir components)."""
if ":" in entry:
raw_path, label = entry.rsplit(":", 1)
else:
raw_path = entry
p = Path(entry)
label = f"{p.parent.name}/{p.name}"
return Path(raw_path), label
def main() -> None:
ap = argparse.ArgumentParser(
description="Per-class ROC curves comparing multiple v2 HyperTower runs.",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__,
)
ap.add_argument(
"--runs", nargs="+", required=True, metavar="PATH[:LABEL]",
help=(
"Mode-level directories to compare, each optionally followed by :label. "
"Path should point to the {eval_mode}/{tower_mode} subdirectory."
),
)
ap.add_argument("--mode", required=True, choices=["binary", "multiclass"])
ap.add_argument("--tag", required=True, help="Output filename prefix.")
ap.add_argument(
"--output-dir", default="analysis_data/roc_plots",
help="Directory for PNG and JSON output (default: analysis_data/roc_plots).",
)
ap.add_argument(
"--class-names", nargs="*", default=["Healthy", "Glaucoma", "Suspect"],
)
ap.add_argument("--shade", action="store_true", help="Shade ±1 SD bands.")
args = ap.parse_args()
out_dir = Path(args.output_dir)
out_dir.mkdir(parents=True, exist_ok=True)
per_model: list[tuple[str, dict]] = []
for entry in args.runs:
mode_dir, label = parse_run_entry(entry)
if not mode_dir.exists():
print(f" WARNING: {mode_dir} not found — skipping.")
continue
tower_mode = mode_dir.name
result = build_mean_curve(mode_dir, tower_mode, args.mode)
if result is None:
print(f" WARNING: no usable folds in {mode_dir} — skipping.")
continue
auc_str = " ".join(
f"class{k} AUC={v['auc_mean']:.3f}±{v['auc_std']:.3f}"
for k, v in result["class_curves"].items()
)
print(f" {label} [{result['probs_stem']}] {auc_str}")
per_model.append((label, result["class_curves"]))
if not per_model:
raise SystemExit("No usable runs — nothing to plot.")
if args.mode == "binary":
classes_to_plot = [1]
out_names = [f"{args.tag}_binary_roc.png"]
titles = ["Binary — Glaucoma (positive class)"]
else:
max_k = max(max(curves.keys()) for _, curves in per_model)
classes_to_plot = list(range(min(3, max_k + 1)))
out_names = [f"{args.tag}_class{k}_roc.png" for k in classes_to_plot]
titles = [
f"Multiclass OVR — "
f"{args.class_names[k] if k < len(args.class_names) else f'class {k}'}"
for k in classes_to_plot
]
out_json: dict = {"tag": args.tag, "mode": args.mode, "figures": []}
for k, out_name, title in zip(classes_to_plot, out_names, titles):
fig, ax = plt.subplots(figsize=(9, 7))
ax.plot([0, 1], [0, 1], linestyle="--", linewidth=1, color="grey")
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(f"{title}\n{args.tag}")
entries = []
for label, curves in per_model:
if k not in curves:
continue
c = curves[k]
ax.plot(
c["fpr"], c["tpr_mean"], linewidth=2,
label=f"{label} (AUC {c['auc_mean']:.3f} ± {c['auc_std']:.3f})",
)
if args.shade:
ax.fill_between(
c["fpr"],
np.clip(c["tpr_mean"] - c["tpr_std"], 0, 1),
np.clip(c["tpr_mean"] + c["tpr_std"], 0, 1),
alpha=0.10,
)
entries.append({
"label": label,
"auc_mean": c["auc_mean"],
"auc_std": c["auc_std"],
})
ax.legend(loc="lower right")
fig.tight_layout()
out_path = out_dir / out_name
fig.savefig(out_path, dpi=160)
plt.close(fig)
print(f" Saved: {out_path}")
out_json["figures"].append({
"class_index": k,
"output_png": str(out_path),
"models": entries,
})
summary_path = out_dir / f"{args.tag}_roc_summary.json"
summary_path.write_text(json.dumps(out_json, indent=2))
print(f" Summary: {summary_path}")
if __name__ == "__main__":
main()
scripts/output_analysis/visualizations/plot_run_roc_v2.py
+286
View File
@@ -0,0 +1,286 @@
#!/usr/bin/env python3
"""
Per-fold and mean OVR ROC plots for a single v2 HyperTower run.
Reads the saved .npy artifacts from a completed run and produces:
- One per-fold ROC figure per class (all folds as individual lines)
- One mean ± SD OVR ROC figure (all classes on the same axes)
Outputs are written to {mode_dir}/plots/.
v2 directory layout expected
-----------------------------
{run_dir}/{eval_mode}/{tower_mode}/
fold0/ y_true.npy probs_fused.npy | probs_bilat.npy | probs_classic.npy | probs_fused_head.npy
fold1/ ...
Usage examples
--------------
# Ensemble binary run
python scripts/output_analysis/visualizations/plot_run_roc_v2.py \\
--run-dir analysis_data/v2_modes_full_40ep_5fold_roi_unet_perimage_refugebuild_holdout \\
--eval-mode binary --tower-mode ensemble
# Bilateral multiclass run
python scripts/output_analysis/visualizations/plot_run_roc_v2.py \\
--run-dir analysis_data/v2_modes_full_40ep_5fold_roi_unet_perimage_refugebuild_holdout \\
--eval-mode multiclass --tower-mode bilateral
# Fused head — explicitly select probs file
python scripts/output_analysis/visualizations/plot_run_roc_v2.py \\
--run-dir analysis_data/v2_ensemble_fused_binary_unet_40ep_5fold_v1 \\
--eval-mode binary --tower-mode ensemble --probs probs_fused_head
"""
from __future__ import annotations
import argparse
from pathlib import Path
import numpy as np
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, auc as sk_auc
# ---------------------------------------------------------------------------
# Probs auto-detection
# ---------------------------------------------------------------------------
_PROBS_PRIORITY: dict[str, list[str]] = {
"ensemble": ["probs_fused_head", "probs_fused"],
"bilateral": ["probs_bilat"],
"single": ["probs_classic"],
"classic": ["probs_classic"],
}
_ALL_PROBS = ["probs_fused_head", "probs_fused", "probs_bilat", "probs_classic"]
def detect_probs_stem(fold_dir: Path, tower_mode: str | None) -> str | None:
priority = _PROBS_PRIORITY.get(tower_mode, _ALL_PROBS) if tower_mode else _ALL_PROBS
for stem in priority:
if (fold_dir / f"{stem}.npy").exists():
return stem
return None
# ---------------------------------------------------------------------------
# Data loading
# ---------------------------------------------------------------------------
def find_fold_dirs(mode_dir: Path) -> list[Path]:
return sorted(
[p for p in mode_dir.iterdir() if p.is_dir() and p.name.startswith("fold")],
key=lambda p: int(p.name.replace("fold", "")),
)
def _find_y_true(fold_dir: Path) -> Path | None:
"""
Return path to y_true.npy for this fold. If missing from fold_dir
(can happen for bilateral-only old runs), fall back to the same fold
index under sibling tower-mode directories (ensemble → single → classic).
Labels are shared across tower modes within the same fold.
"""
local = fold_dir / "y_true.npy"
if local.exists():
return local
fold_name = fold_dir.name # e.g. "fold0"
tower_dir = fold_dir.parent # e.g. .../binary/bilateral
eval_dir = tower_dir.parent # e.g. .../binary
for fallback in ("ensemble", "single", "classic"):
candidate = eval_dir / fallback / fold_name / "y_true.npy"
if candidate.exists():
return candidate
return None
def load_fold(
fold_dir: Path,
probs_stem: str,
eval_mode: str,
) -> tuple[np.ndarray, np.ndarray] | None:
y_path = _find_y_true(fold_dir)
p_path = fold_dir / f"{probs_stem}.npy"
if y_path is None or not p_path.exists():
return None
y = np.load(y_path)
p = np.load(p_path)
if eval_mode == "binary":
mask = np.isin(y, [0, 1])
y, p = y[mask], p[mask]
if p.shape[1] > 2:
p = p[:, :2]
return y, p
# ---------------------------------------------------------------------------
# ROC helpers
# ---------------------------------------------------------------------------
def per_class_roc(y: np.ndarray, p: np.ndarray) -> dict[int, dict]:
"""OVR ROC for each class. Returns {k: {fpr, tpr, auc}}."""
out: dict[int, dict] = {}
for k in range(p.shape[1]):
yb = (y == k).astype(np.uint8)
if yb.sum() == 0 or yb.sum() == len(yb):
continue
fpr, tpr, _ = roc_curve(yb, p[:, k])
out[k] = {"fpr": fpr, "tpr": tpr, "auc": sk_auc(fpr, tpr)}
return out
# ---------------------------------------------------------------------------
# Plotting
# ---------------------------------------------------------------------------
def plot_perfold(
per_fold: list[tuple[int, dict]],
out_dir: Path,
class_names: list[str],
probs_stem: str,
eval_mode: str,
) -> None:
"""One figure per class: each fold as a separate line."""
all_classes = sorted({k for _, curves in per_fold for k in curves})
if eval_mode == "binary":
all_classes = [k for k in all_classes if k == 1]
out_dir.mkdir(parents=True, exist_ok=True)
for k in all_classes:
cname = class_names[k] if k < len(class_names) else f"class_{k}"
fig, ax = plt.subplots(figsize=(9, 7))
ax.plot([0, 1], [0, 1], linestyle="--", linewidth=1, color="grey")
for fold_idx, curves in per_fold:
if k not in curves:
continue
c = curves[k]
auc_val = c["auc"]
ax.plot(c["fpr"], c["tpr"], linewidth=1.5,
label=f"Fold {fold_idx} (AUC {auc_val:.3f})")
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(f"Per-fold ROC — {cname} [{probs_stem}]")
ax.legend(loc="lower right")
fig.tight_layout()
safe = cname.replace(" ", "_")
fig.savefig(out_dir / f"roc_{probs_stem}_{safe}_perfold.png", dpi=160)
plt.close(fig)
print(f" Saved per-fold ROC ({cname})")
def plot_mean_ovr(
per_fold: list[tuple[int, dict]],
out_dir: Path,
class_names: list[str],
probs_stem: str,
eval_mode: str,
) -> None:
"""Mean ± SD OVR ROC — all classes on one figure."""
all_classes = sorted({k for _, curves in per_fold for k in curves})
if eval_mode == "binary":
all_classes = [k for k in all_classes if k == 1]
grid = np.linspace(0, 1, 501)
out_dir.mkdir(parents=True, exist_ok=True)
fig, ax = plt.subplots(figsize=(9, 7))
ax.plot([0, 1], [0, 1], linestyle="--", linewidth=1, color="grey")
for k in all_classes:
cname = class_names[k] if k < len(class_names) else f"class_{k}"
tprs, aucs = [], []
for _, curves in per_fold:
if k not in curves:
continue
c = curves[k]
tprs.append(np.interp(grid, c["fpr"], c["tpr"]))
aucs.append(c["auc"])
if not tprs:
continue
tprs_arr = np.vstack(tprs)
mean = tprs_arr.mean(axis=0)
std = tprs_arr.std(axis=0)
label = f"{cname} (AUC {np.nanmean(aucs):.3f} ± {np.nanstd(aucs):.3f})"
line, = ax.plot(grid, mean, linewidth=2, label=label)
ax.fill_between(grid,
np.clip(mean - std, 0, 1),
np.clip(mean + std, 0, 1),
alpha=0.15, color=line.get_color())
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(f"Mean OVR ROC (± 1 SD) [{probs_stem}]")
ax.legend(loc="lower right")
fig.tight_layout()
out_path = out_dir / f"roc_{probs_stem}_mean_ovr.png"
fig.savefig(out_path, dpi=160)
plt.close(fig)
print(f" Saved mean OVR ROC: {out_path}")
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def main() -> None:
ap = argparse.ArgumentParser(
description="Per-fold and mean OVR ROC plots for a single v2 run.",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__,
)
ap.add_argument("--run-dir", required=True, type=Path,
help="Top-level run directory (e.g. analysis_data/v2_my_run).")
ap.add_argument("--eval-mode", required=True, choices=["binary", "multiclass"])
ap.add_argument("--tower-mode", required=True,
choices=["single", "classic", "ensemble", "bilateral"],
help="Tower mode subdirectory to read from.")
ap.add_argument("--probs", default=None,
help="Probs file stem to use (e.g. probs_fused, probs_bilat, "
"probs_fused_head). Auto-detected if omitted.")
ap.add_argument("--class-names", nargs="*",
default=["Healthy", "Glaucoma", "Suspect"])
args = ap.parse_args()
mode_dir = args.run_dir / args.eval_mode / args.tower_mode
if not mode_dir.exists():
raise SystemExit(f"Directory not found: {mode_dir}")
fold_dirs = find_fold_dirs(mode_dir)
if not fold_dirs:
raise SystemExit(f"No fold subdirectories found in {mode_dir}")
# Determine probs stem
probs_stem = args.probs
if probs_stem is None:
for fd in fold_dirs:
probs_stem = detect_probs_stem(fd, args.tower_mode)
if probs_stem:
break
if probs_stem is None:
raise SystemExit(f"Could not detect a probs file in {mode_dir}/fold*/")
print(f"Using probs: {probs_stem}.npy")
# Load all folds
per_fold: list[tuple[int, dict]] = []
for fd in fold_dirs:
fold_idx = int(fd.name.replace("fold", ""))
result = load_fold(fd, probs_stem, args.eval_mode)
if result is None:
print(f" [skip] fold {fold_idx}: missing y_true or {probs_stem}.npy")
continue
y, p = result
curves = per_class_roc(y, p)
per_fold.append((fold_idx, curves))
auc_str = " ".join(
f"class{k}={v['auc']:.3f}" for k, v in curves.items()
)
print(f" fold {fold_idx}: {auc_str}")
if not per_fold:
raise SystemExit("No usable folds — nothing to plot.")
out_dir = mode_dir / "plots"
plot_perfold(per_fold, out_dir, args.class_names, probs_stem, args.eval_mode)
plot_mean_ovr(per_fold, out_dir, args.class_names, probs_stem, args.eval_mode)
print(f"\nPlots written to {out_dir}")
if __name__ == "__main__":
main()
scripts/output_analysis/visualizations/rebuild_run_best_plots.py
+95 -10
View File
@@ -19,7 +19,7 @@ from types import SimpleNamespace
import matplotlib
import sys
REPO_ROOT = Path(__file__).resolve().parents[1]
REPO_ROOT = Path(__file__).resolve().parents[3]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
matplotlib.use("Agg")
@@ -70,10 +70,34 @@ def load_summary(run_dir: Path) -> dict:
return json.load(fh)
def resolve_data_dir(raw_dir: str) -> str:
"""
Resolve dataset paths saved in legacy cli_args.json.
Older runs often store "ClinicalData"/"FundusImages" relative to a
dataset root, while current repo layout uses "Papila/<dir>".
"""
p = Path(raw_dir)
if p.exists():
return str(p)
candidates = [
REPO_ROOT / p,
REPO_ROOT / "Papila" / p,
]
for cand in candidates:
if cand.exists():
return str(cand)
return str(p)
def prepare_clinical(cli_args: dict, run_dir: Path) -> tuple:
image_dir = resolve_data_dir(cli_args["image_dir"])
clinical_dir = resolve_data_dir(cli_args["clinical_dir"])
clinical = build_papila_clinical(
cli_args["image_dir"],
cli_args["clinical_dir"],
image_dir,
clinical_dir,
cli_args["label_col"],
cli_args["cat_cols"],
n_splits=cli_args["n_splits"],
@@ -103,10 +127,12 @@ def prepare_clinical(cli_args: dict, run_dir: Path) -> tuple:
def build_ht_args(cli_args: dict, fold: int, run_dir: Path, models_dir: Path, holdout_df):
image_dir = resolve_data_dir(cli_args["image_dir"])
clinical_dir = resolve_data_dir(cli_args["clinical_dir"])
# Copy of the training-time namespace so HyperTower can be re-instantiated.
return SimpleNamespace(
image_dir=cli_args["image_dir"],
clinical_dir=cli_args["clinical_dir"],
image_dir=image_dir,
clinical_dir=clinical_dir,
label_col=cli_args["label_col"],
cat_cols=cli_args["cat_cols"],
batch_size=cli_args["batch_size"],
@@ -253,6 +279,14 @@ def choose_head_probs(head: str, probs_f, probs_i, probs_m):
return probs_i
def legacy_probs_suffix(head: str) -> str:
if head == "image":
return "img"
if head == "metadata":
return "md"
return "fused"
def ensure_binary_slice(y_true, *arrays):
mask = np.isin(y_true, [0, 1])
filtered = [y_true[mask]]
@@ -264,8 +298,17 @@ def ensure_binary_slice(y_true, *arrays):
return filtered
def plot_overlays(per_fold_curves, out_dir: Path, class_names: list[str], head: str, suffix: str = ""):
def plot_overlays(
per_fold_curves,
out_dir: Path,
class_names: list[str],
head: str,
eval_mode: str,
suffix: str = "",
):
keys = sorted({k for _, curves in per_fold_curves for k in curves.keys()})
if eval_mode == "binary":
keys = [k for k in keys if k == 1]
if not keys:
return
name_map = {k: (class_names[k] if k < len(class_names) else f"class_{k}") for k in keys}
@@ -293,8 +336,17 @@ def plot_overlays(per_fold_curves, out_dir: Path, class_names: list[str], head:
plt.close(fig)
def plot_mean_sd(per_fold_curves, out_dir: Path, class_names: list[str], head: str, suffix: str = ""):
def plot_mean_sd(
per_fold_curves,
out_dir: Path,
class_names: list[str],
head: str,
eval_mode: str,
suffix: str = "",
):
keys = sorted({k for _, curves in per_fold_curves for k in curves.keys()})
if eval_mode == "binary":
keys = [k for k in keys if k == 1]
if not keys:
return
grid = np.linspace(0, 1, 501)
@@ -363,10 +415,43 @@ def main():
print(f"[skip] Fold {fold_idx}: no best_epoch recorded.")
continue
# Prefer saved fold arrays when available. This avoids reconstructing
# HyperTower for legacy runs whose external weight paths no longer exist.
base = run_dir / f"fold{fold_idx}{file_suffix}"
y_path = Path(f"{base}_y_true.npy")
p_path = Path(f"{base}_probs_{legacy_probs_suffix(head)}.npy")
if y_path.exists() and p_path.exists():
y_true = np.load(y_path)
head_probs = np.load(p_path)
if cli_args["eval_mode"] == "binary" and head_probs.shape[1] >= 2:
head_probs = head_probs[:, :2]
curves = compute_per_class_curves(y_true, head_probs)
per_fold_curves.append((fold_idx, curves))
try:
if head_probs.shape[1] > 2:
fold_auc = roc_auc_score(y_true, head_probs, multi_class="ovr", average="macro")
else:
target_scores = head_probs[:, 1] if head_probs.shape[1] > 1 else head_probs[:, 0]
fold_auc = roc_auc_score(y_true, target_scores)
fold_aucs.append(fold_auc)
print(
f"[info] Fold {fold_idx}: using saved arrays "
f"({y_path.name}, {p_path.name}), AUC={fold_auc:.4f}"
)
except Exception:
print(
f"[warning] Fold {fold_idx}: using saved arrays "
f"({y_path.name}, {p_path.name}) but AUC failed."
)
continue
fold_models_dir = base_models_dir / f"fold{fold_idx}"
best_checkpoint = fold_models_dir / "model_best.pt"
if not best_checkpoint.exists():
print(f"[warning] Fold {fold_idx}: missing model_best.pt at {best_checkpoint}")
print(
f"[warning] Fold {fold_idx}: missing model_best.pt at {best_checkpoint} "
f"and missing fallback arrays {y_path.name}/{p_path.name}"
)
continue
ht_args = build_ht_args(cli_args, fold_idx, run_dir, fold_models_dir, holdout_df)
@@ -435,8 +520,8 @@ def main():
raise SystemExit("No folds processed; nothing to plot.")
plots_dir = run_dir / "plots"
plot_overlays(per_fold_curves, plots_dir, class_names, head, file_suffix)
plot_mean_sd(per_fold_curves, plots_dir, class_names, head, file_suffix)
plot_overlays(per_fold_curves, plots_dir, class_names, head, cli_args["eval_mode"], file_suffix)
plot_mean_sd(per_fold_curves, plots_dir, class_names, head, cli_args["eval_mode"], file_suffix)
if fold_aucs:
print(f"[info] {head} head mean AUC across folds: {np.mean(fold_aucs):.4f} ± {np.std(fold_aucs):.4f}")