ryan/hypertower
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update 3-19

Browse Source
This commit is contained in:
rpotter6298
2026-03-19 11:18:58 +01:00
parent 7ea85d5426
commit 786457b30d
35 changed files with 4019 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/output_analysis/aggregate_10x5cv.py
+496
View File
@@ -0,0 +1,496 @@
#!/usr/bin/env python3
"""
Aggregate and visualise results from a 10× repeated 5-fold CV run.
Reads probs / y_true from every rep/fold directory, computes per-fold
metrics, and produces:
outputs/
fold_metrics.csv — one row per (rep, fold, eval_mode)
rep_metrics.csv — one row per (rep, eval_mode): mean over 5 folds
overall_summary.txt — mean ± SD and 95% CI printed to console + file
{eval_mode}_auc_violin.png
{eval_mode}_roc_mean.png — mean ± 1 SD OVR ROC (all classes or class 1)
{eval_mode}_holdout_roc_mean.png
Holdout metrics are extracted from the rep-level predictions.npz using the
best_epoch recorded in summary.json, ensemble-averaged over od_fused + os_fused
heads, giving 50 fold-level holdout AUC values (5 folds × 10 reps).
Usage
-----
python scripts/output_analysis/aggregate_10x5cv.py \
--run-root analysis_data/pipeline_10x5 \
--eval-modes binary multiclass \
--out analysis_data/pipeline_10x5/aggregate
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.metrics import roc_auc_score, accuracy_score, roc_curve, auc as sk_auc
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
_CLASS_NAMES = {
"binary": ["Healthy", "Glaucoma"],
"multiclass": ["Healthy", "Glaucoma", "Suspect"],
}
# Probe files in preference order (first found wins)
# probs_fused = simple OD/OS softmax average (ensemble head — primary metric)
# probs_fused_head = learned logit-level fusion head (worse on average; kept as fallback)
_PROBS_PRIORITY = ["probs_fused.npy", "probs_fused_head.npy"]
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _find_probs(fold_dir: Path) -> Path | None:
for name in _PROBS_PRIORITY:
p = fold_dir / name
if p.exists():
return p
return None
def _load_holdout_summary(mode_dir: Path) -> dict | None:
"""
Read rep-level holdout metrics from summary.json.
Returns dict with keys auc_mean, auc_std, acc_mean (may be None if missing).
"""
summary_path = mode_dir / "summary.json"
if not summary_path.exists():
return None
try:
summary = json.loads(summary_path.read_text())
return summary.get("mode_summary", {}).get("ensemble_holdout")
except Exception:
return None
def _auc_macro(y: np.ndarray, p: np.ndarray, num_classes: int) -> float:
try:
if num_classes == 2:
return float(roc_auc_score(y, p[:, 1]))
return float(roc_auc_score(y, p, multi_class="ovr", average="macro"))
except Exception:
return float("nan")
def _per_class_roc(y: np.ndarray, p: np.ndarray) -> dict[int, dict]:
out: dict[int, dict] = {}
for k in range(p.shape[1]):
yb = (y == k).astype(np.uint8)
if yb.sum() == 0 or yb.sum() == len(yb):
continue
fpr, tpr, _ = roc_curve(yb, p[:, k])
out[k] = {"fpr": fpr, "tpr": tpr, "auc": sk_auc(fpr, tpr)}
return out
def _ci95(values: np.ndarray) -> tuple[float, float]:
"""95% CI via t-distribution (two-sided)."""
from scipy import stats as scipy_stats
if len(values) < 2:
return (float("nan"), float("nan"))
ci = scipy_stats.t.interval(0.95, df=len(values) - 1,
loc=np.mean(values), scale=scipy_stats.sem(values))
return float(ci[0]), float(ci[1])
# ---------------------------------------------------------------------------
# Data loading
# ---------------------------------------------------------------------------
def load_all_folds(run_root: Path, eval_modes: list[str]) -> pd.DataFrame:
rows = []
rep_dirs = sorted(
[d for d in run_root.iterdir() if d.is_dir() and d.name.startswith("rep")],
key=lambda d: d.name,
)
if not rep_dirs:
raise SystemExit(f"No rep* directories found in {run_root}")
for rep_dir in rep_dirs:
for eval_mode in eval_modes:
tower_mode = "ensemble"
mode_dir = rep_dir / eval_mode / tower_mode
if not mode_dir.exists():
print(f" [skip] {mode_dir} not found")
continue
num_classes = 2 if eval_mode == "binary" else 3
fold_dirs = sorted(
[d for d in mode_dir.iterdir() if d.is_dir() and d.name.startswith("fold")],
key=lambda d: int(d.name[4:]),
)
for fold_dir in fold_dirs:
fold_idx = int(fold_dir.name[4:])
y_path = fold_dir / "y_true.npy"
p_path = _find_probs(fold_dir)
if y_path is None or not y_path.exists() or p_path is None:
print(f" [skip] {rep_dir.name}/{eval_mode}/fold{fold_idx}: missing files")
continue
y = np.load(y_path)
p = np.load(p_path)
if eval_mode == "binary":
mask = np.isin(y, [0, 1])
y, p = y[mask], p[mask]
if p.shape[1] > 2:
p = p[:, :2]
auc_macro = _auc_macro(y, p, num_classes)
acc = float(accuracy_score(y, p.argmax(1)))
row = {
"rep": rep_dir.name,
"fold": fold_idx,
"eval_mode": eval_mode,
"probs_file": p_path.name,
"auc_macro": auc_macro,
"acc": acc,
"n": len(y),
}
# Per-class AUC
for k in range(num_classes):
yb = (y == k).astype(np.uint8)
if yb.sum() > 0 and yb.sum() < len(yb):
try:
row[f"auc_class{k}"] = float(roc_auc_score(yb, p[:, k]))
except Exception:
row[f"auc_class{k}"] = float("nan")
else:
row[f"auc_class{k}"] = float("nan")
rows.append(row)
# ---- holdout metrics from rep-level summary.json ----
# Holdout probs are not stored per-fold; only aggregated stats are saved.
# We attach the rep-level mean to each fold row (same value repeated),
# and also add a single rep-level summary row (fold=-1).
hld_summary = _load_holdout_summary(mode_dir)
if hld_summary:
hld_auc = hld_summary.get("auc_mean", float("nan"))
hld_auc_std = hld_summary.get("auc_std", float("nan"))
hld_acc = hld_summary.get("acc_mean", float("nan"))
for row in rows:
if row["rep"] == rep_dir.name and row["eval_mode"] == eval_mode:
row["hld_auc_macro"] = hld_auc
row["hld_acc"] = hld_acc
# Also store a rep-level holdout row (fold=-1) for direct rep-level analysis
rows.append({
"rep": rep_dir.name,
"fold": -1,
"eval_mode": eval_mode,
"probs_file": "summary.json",
"auc_macro": float("nan"),
"acc": float("nan"),
"n": float("nan"),
"hld_auc_macro": hld_auc,
"hld_auc_std_within_rep": hld_auc_std,
"hld_acc": hld_acc,
})
return pd.DataFrame(rows)
# ---------------------------------------------------------------------------
# Plotting
# ---------------------------------------------------------------------------
def _violin(fold_df: pd.DataFrame, eval_mode: str, out_dir: Path) -> None:
sub = fold_df[fold_df["eval_mode"] == eval_mode].copy()
num_classes = 2 if eval_mode == "binary" else 3
class_names = _CLASS_NAMES[eval_mode]
auc_cols = ["auc_macro"] + [f"auc_class{k}" for k in range(num_classes)]
labels = ["Macro AUC"] + [f"AUC {class_names[k]}" for k in range(num_classes)]
present = [(c, l) for c, l in zip(auc_cols, labels) if c in sub.columns]
data = [sub[c].dropna().values for c, _ in present]
labels = [l for _, l in present]
fig, ax = plt.subplots(figsize=(max(6, 2 * len(data)), 5))
parts = ax.violinplot(data, showmedians=True, showextrema=True)
for pc in parts["bodies"]:
pc.set_alpha(0.7)
# Overlay individual rep means
rep_means = sub.groupby("rep")[auc_cols[0]].mean().values
ax.scatter(np.ones(len(rep_means)), rep_means, zorder=3,
color="k", s=18, alpha=0.6, label="rep mean")
ax.set_xticks(range(1, len(labels) + 1))
ax.set_xticklabels(labels, rotation=15, ha="right")
ax.set_ylabel("AUC")
ax.set_title(f"AUC distribution — {eval_mode} (10 × 5-fold, n={len(sub)})")
ax.set_ylim(max(0, sub[auc_cols[0]].min() - 0.05), 1.02)
ax.grid(True, axis="y", linewidth=0.4, alpha=0.5)
ax.legend(fontsize=8)
fig.tight_layout()
path = out_dir / f"{eval_mode}_auc_violin.png"
fig.savefig(path, dpi=160, bbox_inches="tight")
plt.close(fig)
print(f" Saved: {path}")
def _mean_roc(fold_df: pd.DataFrame, eval_mode: str,
run_root: Path, out_dir: Path) -> None:
"""Mean ± 1 SD OVR ROC across all 50 folds."""
sub = fold_df[fold_df["eval_mode"] == eval_mode]
num_classes = 2 if eval_mode == "binary" else 3
class_names = _CLASS_NAMES[eval_mode]
# Classes to plot (binary: class 1 only)
plot_classes = [1] if eval_mode == "binary" else list(range(num_classes))
grid = np.linspace(0, 1, 501)
fig, ax = plt.subplots(figsize=(9, 7))
ax.plot([0, 1], [0, 1], linestyle="--", linewidth=1, color="grey")
for k in plot_classes:
tprs, aucs = [], []
for _, row in sub.iterrows():
rep_dir = run_root / row["rep"]
fold_dir = rep_dir / eval_mode / "ensemble" / f"fold{int(row['fold'])}"
y_path = fold_dir / "y_true.npy"
p_path = _find_probs(fold_dir)
if not y_path.exists() or p_path is None:
continue
y = np.load(y_path)
p = np.load(p_path)
if eval_mode == "binary":
mask = np.isin(y, [0, 1])
y, p = y[mask], p[mask]
if p.shape[1] > 2:
p = p[:, :2]
yb = (y == k).astype(np.uint8)
if yb.sum() == 0 or yb.sum() == len(yb):
continue
fpr, tpr, _ = roc_curve(yb, p[:, k])
tprs.append(np.interp(grid, fpr, tpr))
aucs.append(sk_auc(fpr, tpr))
if not tprs:
continue
arr = np.vstack(tprs)
mean = arr.mean(0)
std = arr.std(0)
cname = class_names[k]
lbl = f"{cname} AUC {np.nanmean(aucs):.3f} ± {np.nanstd(aucs):.3f}"
line, = ax.plot(grid, mean, linewidth=2, label=lbl)
ax.fill_between(grid,
np.clip(mean - std, 0, 1),
np.clip(mean + std, 0, 1),
alpha=0.15, color=line.get_color())
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(f"Mean ± 1 SD OVR ROC — {eval_mode} (10 × 5-fold)")
ax.legend(loc="lower right", fontsize=9)
fig.tight_layout()
path = out_dir / f"{eval_mode}_roc_mean.png"
fig.savefig(path, dpi=160, bbox_inches="tight")
plt.close(fig)
print(f" Saved: {path}")
def _holdout_stability(fold_df: pd.DataFrame, eval_mode: str, out_dir: Path) -> None:
"""Bar chart of per-rep holdout AUC (mean across folds within rep ± within-rep SD)."""
# use the rep-level rows (fold == -1) which have hld_auc_std_within_rep
rep_rows = fold_df[(fold_df["eval_mode"] == eval_mode) & (fold_df["fold"] == -1)].copy()
if rep_rows.empty or "hld_auc_macro" not in rep_rows.columns:
print(f" [skip] no holdout data for {eval_mode}")
return
rep_rows = rep_rows.sort_values("rep")
fig, ax = plt.subplots(figsize=(max(6, len(rep_rows) * 0.9), 4))
x = np.arange(len(rep_rows))
yerr = rep_rows.get("hld_auc_std_within_rep", pd.Series([0]*len(rep_rows))).fillna(0).values
ax.bar(x, rep_rows["hld_auc_macro"].values, yerr=yerr,
capsize=4, color="darkorange", alpha=0.8)
grand_mean = rep_rows["hld_auc_macro"].mean()
ax.axhline(grand_mean, linestyle="--", color="crimson",
linewidth=1.2, label=f"grand mean = {grand_mean:.3f}")
ax.set_xticks(x)
ax.set_xticklabels(rep_rows["rep"].values, rotation=30, ha="right")
ax.set_ylabel("Holdout macro AUC (mean ± within-rep SD)")
ax.set_title(f"Per-rep holdout stability — {eval_mode}")
ymin = max(0, rep_rows["hld_auc_macro"].min() - 0.05)
ax.set_ylim(ymin, 1.02)
ax.legend(fontsize=9)
ax.grid(True, axis="y", linewidth=0.4, alpha=0.5)
fig.tight_layout()
path = out_dir / f"{eval_mode}_holdout_stability.png"
fig.savefig(path, dpi=160, bbox_inches="tight")
plt.close(fig)
print(f" Saved: {path}")
def _rep_stability(fold_df: pd.DataFrame, eval_mode: str, out_dir: Path) -> None:
"""Bar chart of per-rep mean macro AUC with ± 1 SD error bars."""
sub = fold_df[fold_df["eval_mode"] == eval_mode]
rep_stats = sub.groupby("rep")["auc_macro"].agg(["mean", "std"]).reset_index()
rep_stats = rep_stats.sort_values("rep")
fig, ax = plt.subplots(figsize=(max(6, len(rep_stats) * 0.9), 4))
x = np.arange(len(rep_stats))
ax.bar(x, rep_stats["mean"], yerr=rep_stats["std"],
capsize=4, color="steelblue", alpha=0.8)
ax.axhline(rep_stats["mean"].mean(), linestyle="--", color="crimson",
linewidth=1.2, label=f"grand mean = {rep_stats['mean'].mean():.3f}")
ax.set_xticks(x)
ax.set_xticklabels(rep_stats["rep"], rotation=30, ha="right")
ax.set_ylabel("Mean macro AUC (5 folds)")
ax.set_title(f"Per-rep stability — {eval_mode}")
ymin = max(0, rep_stats["mean"].min() - rep_stats["std"].max() - 0.02)
ax.set_ylim(ymin, 1.02)
ax.legend(fontsize=9)
ax.grid(True, axis="y", linewidth=0.4, alpha=0.5)
fig.tight_layout()
path = out_dir / f"{eval_mode}_rep_stability.png"
fig.savefig(path, dpi=160, bbox_inches="tight")
plt.close(fig)
print(f" Saved: {path}")
# ---------------------------------------------------------------------------
# Summary text
# ---------------------------------------------------------------------------
def _print_summary(fold_df: pd.DataFrame, eval_modes: list[str]) -> str:
lines = ["=" * 60, "10 × 5-fold CV — aggregate summary", "=" * 60]
for eval_mode in eval_modes:
sub = fold_df[(fold_df["eval_mode"] == eval_mode) & (fold_df["fold"] >= 0)]
if sub.empty:
continue
num_classes = 2 if eval_mode == "binary" else 3
class_names = _CLASS_NAMES[eval_mode]
lines.append(f"\n--- {eval_mode.upper()} ---")
lines.append(f" n_folds = {len(sub)}")
lines.append(" [Validation]")
for col, label in [("auc_macro", "Macro AUC"), ("acc", "Accuracy")]:
if col not in sub.columns:
continue
vals = sub[col].dropna().values
ci_lo, ci_hi = _ci95(vals)
lines.append(
f" {label:18s}: {vals.mean():.4f} ± {vals.std():.4f}"
f" 95% CI [{ci_lo:.4f}, {ci_hi:.4f}]"
)
for k in range(num_classes):
col = f"auc_class{k}"
if col not in sub.columns:
continue
vals = sub[col].dropna().values
if len(vals) == 0:
continue
ci_lo, ci_hi = _ci95(vals)
lines.append(
f" AUC {class_names[k]:12s}: {vals.mean():.4f} ± {vals.std():.4f}"
f" 95% CI [{ci_lo:.4f}, {ci_hi:.4f}]"
)
# Between-rep variance (val)
rep_means = sub.groupby("rep")["auc_macro"].mean().values
lines.append(
f" Rep-mean AUC (n={len(rep_means)}): "
f"{rep_means.mean():.4f} ± {rep_means.std():.4f}"
f" (between-rep SD = {rep_means.std():.4f})"
)
# Holdout — use rep-level rows (fold == -1)
rep_hld = fold_df[
(fold_df["eval_mode"] == eval_mode) &
(fold_df["fold"] == -1) &
fold_df["hld_auc_macro"].notna()
]["hld_auc_macro"].values if "hld_auc_macro" in fold_df.columns else np.array([])
if len(rep_hld) > 0:
lines.append(" [Holdout] (rep-level means, n_reps={})".format(len(rep_hld)))
ci_lo, ci_hi = _ci95(rep_hld)
lines.append(
f" {'Macro AUC':18s}: {rep_hld.mean():.4f} ± {rep_hld.std():.4f}"
f" 95% CI [{ci_lo:.4f}, {ci_hi:.4f}]"
)
lines.append("=" * 60)
return "\n".join(lines)
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def main() -> None:
ap = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
ap.add_argument("--run-root", default="analysis_data/pipeline_10x5",
help="Root directory containing rep* sub-directories.")
ap.add_argument("--eval-modes", nargs="+",
choices=["binary", "multiclass"],
default=["binary", "multiclass"])
ap.add_argument("--out", default=None,
help="Output directory for plots and CSVs "
"(default: {run-root}/aggregate).")
args = ap.parse_args()
run_root = Path(args.run_root)
out_dir = Path(args.out) if args.out else run_root / "aggregate"
out_dir.mkdir(parents=True, exist_ok=True)
print("Loading fold data...")
fold_df = load_all_folds(run_root, args.eval_modes)
if fold_df.empty:
raise SystemExit("No data loaded — check --run-root.")
fold_df.to_csv(out_dir / "fold_metrics.csv", index=False)
print(f" Saved fold_metrics.csv ({len(fold_df)} rows)")
rep_df = (fold_df.groupby(["rep", "eval_mode"])
[["auc_macro", "acc"] +
[c for c in fold_df.columns if c.startswith("auc_class")]]
.mean()
.reset_index())
rep_df.to_csv(out_dir / "rep_metrics.csv", index=False)
print(f" Saved rep_metrics.csv ({len(rep_df)} rows)")
summary_text = _print_summary(fold_df, args.eval_modes)
print("\n" + summary_text)
(out_dir / "overall_summary.txt").write_text(summary_text + "\n")
print(f"\n Saved overall_summary.txt")
print("\nGenerating plots...")
for eval_mode in args.eval_modes:
if fold_df[fold_df["eval_mode"] == eval_mode].empty:
continue
_violin(fold_df, eval_mode, out_dir)
_mean_roc(fold_df, eval_mode, run_root, out_dir)
_rep_stability(fold_df, eval_mode, out_dir)
_holdout_stability(fold_df, eval_mode, out_dir)
print(f"\nAll outputs written to: {out_dir}")
if __name__ == "__main__":
main()