update 3-19
This commit is contained in:
rpotter6298
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Assemble the 10×5 aggregate comparison panel.
|
||||
|
||||
Layout (3 rows × 2 cols):
|
||||
col 0 = binary, col 1 = multiclass
|
||||
|
||||
row 0: mean ROC curve
|
||||
row 1: rep stability
|
||||
row 2: holdout stability
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/visualizations/build_10x5_aggregate_panel.py \
|
||||
--run-dir analysis_data/pipeline_10x5 \
|
||||
--out analysis_data/pipeline_10x5/aggregate/aggregate_panel.png
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.image as mpimg
|
||||
|
||||
|
||||
# (row, col, rel_path, label)
|
||||
CELLS = [
|
||||
(0, 0, "aggregate/binary_roc_mean.png", "Binary — Mean ROC"),
|
||||
(0, 1, "aggregate/multiclass_roc_mean.png", "Multiclass — Mean ROC"),
|
||||
(1, 0, "aggregate/binary_rep_stability.png", "Binary — Rep stability"),
|
||||
(1, 1, "aggregate/multiclass_rep_stability.png", "Multiclass — Rep stability"),
|
||||
(2, 0, "aggregate/binary_holdout_stability.png", "Binary — Holdout stability"),
|
||||
(2, 1, "aggregate/multiclass_holdout_stability.png", "Multiclass — Holdout stability"),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--run-dir", default="analysis_data/pipeline_10x5")
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
run_dir = Path(args.run_dir)
|
||||
out = Path(args.out) if args.out else run_dir / "aggregate" / "aggregate_panel.png"
|
||||
|
||||
fig = plt.figure(figsize=(16, 18))
|
||||
gs = fig.add_gridspec(3, 2, hspace=0.06, wspace=0.04)
|
||||
|
||||
for row, col, rel, label in CELLS:
|
||||
ax = fig.add_subplot(gs[row, col])
|
||||
img = mpimg.imread(str(run_dir / rel))
|
||||
ax.imshow(img)
|
||||
ax.axis("off")
|
||||
ax.set_title(label, fontsize=11, pad=5)
|
||||
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Assemble the 10×5 training dynamics panel.
|
||||
|
||||
Layout: 3-row × 4-col gridspec with spanning
|
||||
|
||||
Top 2×2 (each cell spans 2 cols):
|
||||
row 0, cols 0-1: binary early stopping sweep
|
||||
row 0, cols 2-3: multiclass early stopping sweep
|
||||
row 1, cols 0-1: binary cost of stopping early
|
||||
row 1, cols 2-3: multiclass cost of stopping early
|
||||
|
||||
Bottom row of 4 (one col each):
|
||||
row 2, col 0: binary holdout AUC by epoch
|
||||
row 2, col 1: binary val−holdout gap (val-adjusted)
|
||||
row 2, col 2: multiclass holdout AUC by epoch
|
||||
row 2, col 3: multiclass val−holdout gap (val-adjusted)
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/visualizations/build_10x5_training_panel.py \
|
||||
--run-dir analysis_data/pipeline_10x5 \
|
||||
--out analysis_data/pipeline_10x5/aggregate/training_dynamics_panel.png
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.image as mpimg
|
||||
|
||||
|
||||
# (row, col_start, col_end, rel_path, label)
|
||||
# col_end is exclusive slice — use None for single cell
|
||||
CELLS = [
|
||||
# ---- top 2×2: early stopping (each spans 2 cols) ----
|
||||
(0, 0, 2, "binary/ensemble/plots/early_stopping_sweep_fused.png",
|
||||
"Binary — Early stopping sweep"),
|
||||
(0, 2, 4, "multiclass/ensemble/plots/early_stopping_sweep_fused.png",
|
||||
"Multiclass — Early stopping sweep"),
|
||||
(1, 0, 2, "binary/ensemble/plots/early_stopping_sweep_fused_inverted.png",
|
||||
"Binary — Cost of stopping early"),
|
||||
(1, 2, 4, "multiclass/ensemble/plots/early_stopping_sweep_fused_inverted_tol0.002.png",
|
||||
"Multiclass — Cost of stopping early (CI tol=0.002)"),
|
||||
# ---- bottom row of 4: holdout epoch curves (single col each) ----
|
||||
(2, 0, 1, "binary/ensemble/plots/holdout_epoch_curves_fused.png",
|
||||
"Binary — Holdout AUC by epoch"),
|
||||
(2, 1, 2, "binary/ensemble/plots/holdout_epoch_curves_fused_delta_adj.png",
|
||||
"Binary — Val−Holdout gap (val-adjusted)"),
|
||||
(2, 2, 3, "multiclass/ensemble/plots/holdout_epoch_curves_fused.png",
|
||||
"Multiclass — Holdout AUC by epoch"),
|
||||
(2, 3, 4, "multiclass/ensemble/plots/holdout_epoch_curves_fused_delta_adj.png",
|
||||
"Multiclass — Val−Holdout gap (val-adjusted)"),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--run-dir", default="analysis_data/pipeline_10x5")
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
run_dir = Path(args.run_dir)
|
||||
out = Path(args.out) if args.out else run_dir / "aggregate" / "training_dynamics_panel.png"
|
||||
|
||||
fig = plt.figure(figsize=(22, 16))
|
||||
gs = fig.add_gridspec(3, 4, height_ratios=[1, 1, 0.75], hspace=0.08, wspace=0.04)
|
||||
|
||||
for row, col_start, col_end, rel, label in CELLS:
|
||||
ax = fig.add_subplot(gs[row, col_start:col_end])
|
||||
img = mpimg.imread(str(run_dir / rel))
|
||||
ax.imshow(img)
|
||||
ax.axis("off")
|
||||
ax.set_title(label, fontsize=11, pad=5)
|
||||
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Assemble a 2x2 fusion-head comparison panel from pipeline_nocrop.
|
||||
|
||||
Layout:
|
||||
[binary ensemble ROC] [binary fusion-head explainability summary]
|
||||
[multiclass ensemble ROC][multiclass fusion-head explainability summary]
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/visualizations/build_fusion_head_panel.py \
|
||||
--run-dir analysis_data/pipeline_nocrop \
|
||||
--out analysis_data/pipeline_nocrop/fusion_head_comparison_panel.png
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.image as mpimg
|
||||
import numpy as np
|
||||
|
||||
|
||||
ROC_CELLS = [
|
||||
# (row, col, rel_path, label)
|
||||
(0, 0, "binary/ensemble/plots/roc_probs_fused_mean_ovr.png",
|
||||
"Binary — Ensemble"),
|
||||
(0, 1, "binary/ensemble/plots/roc_probs_fused_head_mean_ovr.png",
|
||||
"Binary — Fusion Head"),
|
||||
(1, 0, "multiclass/ensemble/plots/roc_probs_fused_mean_ovr.png",
|
||||
"Multiclass — Ensemble"),
|
||||
(1, 1, "multiclass/ensemble/plots/roc_probs_fused_head_mean_ovr.png",
|
||||
"Multiclass — Fusion Head"),
|
||||
]
|
||||
|
||||
EXPL_ROWS = [
|
||||
# (row_in_grid, rel_path, label)
|
||||
(2, "binary/ensemble/explainability_fusion_summary_val_fused_head.png",
|
||||
"Binary — Fusion Head events (val)"),
|
||||
(3, "multiclass/ensemble/explainability_fusion_summary_val_fused_head.png",
|
||||
"Multiclass — Fusion Head events (val)"),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--run-dir", default="analysis_data/pipeline_nocrop")
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
run_dir = Path(args.run_dir)
|
||||
out = Path(args.out) if args.out else run_dir / "fusion_head_comparison_panel.png"
|
||||
|
||||
# load all images
|
||||
roc_imgs = {(r, c): (mpimg.imread(str(run_dir / rel)), lbl)
|
||||
for r, c, rel, lbl in ROC_CELLS}
|
||||
expl_imgs = [(mpimg.imread(str(run_dir / rel)), lbl)
|
||||
for _, rel, lbl in EXPL_ROWS]
|
||||
|
||||
# 4-row grid: rows 0-1 are the 2×2 ROC square; rows 2-3 are full-width explainability
|
||||
fig = plt.figure(figsize=(14, 20))
|
||||
gs = fig.add_gridspec(
|
||||
4, 2,
|
||||
height_ratios=[1, 1, 0.6, 0.6],
|
||||
hspace=0.06,
|
||||
wspace=0.04,
|
||||
)
|
||||
|
||||
# ROC cells (2×2)
|
||||
for row, col, _, _ in ROC_CELLS:
|
||||
ax = fig.add_subplot(gs[row, col])
|
||||
img, label = roc_imgs[(row, col)]
|
||||
ax.imshow(img)
|
||||
ax.axis("off")
|
||||
ax.set_title(label, fontsize=11, pad=5)
|
||||
|
||||
# Explainability rows (span both columns)
|
||||
for grid_row, (img, label) in zip([2, 3], expl_imgs):
|
||||
ax = fig.add_subplot(gs[grid_row, :])
|
||||
ax.imshow(img)
|
||||
ax.axis("off")
|
||||
ax.set_title(label, fontsize=11, pad=5)
|
||||
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Assemble a metadata-explainability comparison panel.
|
||||
|
||||
Layout (2 rows × 4 cols):
|
||||
row 0 = binary, row 1 = multiclass
|
||||
|
||||
col 0: single md_importance
|
||||
col 1: ensemble md_importance
|
||||
col 2: nocrop ROC
|
||||
col 3: excl_phakic_axial ROC
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/visualizations/build_md_explainability_panel.py \
|
||||
--nocrop-dir analysis_data/pipeline_nocrop \
|
||||
--excl-dir analysis_data/pipeline_nocrop_excl_phakic_axial \
|
||||
--out analysis_data/pipeline_nocrop/md_explainability_panel.png
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.image as mpimg
|
||||
|
||||
|
||||
# (row, col, dir_key, rel_path, label)
|
||||
# dir_key: "nocrop" or "excl"
|
||||
CELLS = [
|
||||
# ---- binary row (row 0) ----
|
||||
(0, 0, "nocrop", "binary/single/explainability_md_importance_summary.png",
|
||||
"Binary — Single"),
|
||||
(0, 1, "nocrop", "binary/ensemble/explainability_md_importance_summary.png",
|
||||
"Binary — Ensemble"),
|
||||
(0, 2, "nocrop", "binary/ensemble/plots/roc_probs_fused_mean_ovr.png",
|
||||
"Binary — nocrop ROC"),
|
||||
(0, 3, "excl", "binary/ensemble/plots/roc_probs_fused_mean_ovr.png",
|
||||
"Binary — excl phakic+axial ROC"),
|
||||
# ---- multiclass row (row 1) ----
|
||||
(1, 0, "nocrop", "multiclass/single/explainability_md_importance_summary.png",
|
||||
"Multiclass — Single"),
|
||||
(1, 1, "nocrop", "multiclass/ensemble/explainability_md_importance_summary.png",
|
||||
"Multiclass — Ensemble"),
|
||||
(1, 2, "nocrop", "multiclass/ensemble/plots/roc_probs_fused_mean_ovr.png",
|
||||
"Multiclass — nocrop ROC"),
|
||||
(1, 3, "excl", "multiclass/ensemble/plots/roc_probs_fused_mean_ovr.png",
|
||||
"Multiclass — excl phakic+axial ROC"),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--nocrop-dir", default="analysis_data/pipeline_nocrop")
|
||||
ap.add_argument("--excl-dir", default="analysis_data/pipeline_nocrop_excl_phakic_axial")
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
nocrop_dir = Path(args.nocrop_dir)
|
||||
excl_dir = Path(args.excl_dir)
|
||||
out = Path(args.out) if args.out else nocrop_dir / "md_explainability_panel.png"
|
||||
|
||||
fig = plt.figure(figsize=(24, 12))
|
||||
gs = fig.add_gridspec(
|
||||
2, 4,
|
||||
hspace=0.08,
|
||||
wspace=0.04,
|
||||
)
|
||||
|
||||
dirs = {"nocrop": nocrop_dir, "excl": excl_dir}
|
||||
|
||||
for row, col, dir_key, rel, label in CELLS:
|
||||
ax = fig.add_subplot(gs[row, col])
|
||||
img = mpimg.imread(str(dirs[dir_key] / rel))
|
||||
ax.imshow(img)
|
||||
ax.axis("off")
|
||||
ax.set_title(label, fontsize=11, pad=5)
|
||||
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Simulate early stopping at each epoch N and show what val/holdout AUC
|
||||
you would have gotten if you stopped there.
|
||||
|
||||
For each fold and each candidate stopping epoch N:
|
||||
- Find the epoch <= N with the highest val AUC (checkpoint selection)
|
||||
- Record the val AUC and holdout AUC at that epoch
|
||||
|
||||
Then plot mean ± std across all folds as a function of N.
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/visualizations/plot_early_stopping_sweep.py \
|
||||
--run-dir analysis_data/pipeline_10x5 \
|
||||
--eval-mode binary \
|
||||
--tower-mode ensemble \
|
||||
--head fused
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
HEAD_COL = {
|
||||
"fused": ("ensemble_val_auc", "ensemble_holdout_auc"),
|
||||
"img": ("ensemble_val_auc_img", "ensemble_holdout_auc_img"),
|
||||
"md": ("ensemble_val_auc_md", "ensemble_holdout_auc_md"),
|
||||
"classic": ("classic_val_auc", "classic_holdout_auc"),
|
||||
}
|
||||
|
||||
|
||||
def load_fold_logs(run_dir: Path, eval_mode: str, tower_mode: str,
|
||||
val_col: str, hld_col: str):
|
||||
logs = []
|
||||
for rep_dir in sorted(run_dir.glob("rep*")):
|
||||
mode_dir = rep_dir / eval_mode / tower_mode
|
||||
if not mode_dir.exists():
|
||||
continue
|
||||
for fd in sorted(
|
||||
[d for d in mode_dir.iterdir() if d.is_dir() and d.name.startswith("fold")],
|
||||
key=lambda p: int(p.name.replace("fold", "")),
|
||||
):
|
||||
log = fd / "epoch_log.csv"
|
||||
if not log.exists():
|
||||
continue
|
||||
df = pd.read_csv(log)
|
||||
if val_col not in df.columns or hld_col not in df.columns:
|
||||
continue
|
||||
df = df[["epoch", val_col, hld_col]].dropna()
|
||||
logs.append(df.reset_index(drop=True))
|
||||
return logs
|
||||
|
||||
|
||||
def sweep(logs: list[pd.DataFrame], val_col: str, hld_col: str):
|
||||
max_epoch = max(df["epoch"].max() for df in logs)
|
||||
epochs = np.arange(1, int(max_epoch) + 1)
|
||||
|
||||
val_mat = np.full((len(logs), len(epochs)), np.nan)
|
||||
hld_mat = np.full((len(logs), len(epochs)), np.nan)
|
||||
|
||||
for i, df in enumerate(logs):
|
||||
for j, n in enumerate(epochs):
|
||||
window = df[df["epoch"] <= n]
|
||||
if window.empty:
|
||||
continue
|
||||
best_idx = window[val_col].idxmax()
|
||||
val_mat[i, j] = window.loc[best_idx, val_col]
|
||||
hld_mat[i, j] = window.loc[best_idx, hld_col]
|
||||
|
||||
return epochs, val_mat, hld_mat
|
||||
|
||||
|
||||
def plot(epochs, val_mat, hld_mat, out_path: Path, title: str, inverted: bool = False, ci_tol: float = 0.0):
|
||||
val_mean = np.nanmean(val_mat, axis=0)
|
||||
val_std = np.nanstd(val_mat, axis=0)
|
||||
hld_mean = np.nanmean(hld_mat, axis=0)
|
||||
hld_std = np.nanstd(hld_mat, axis=0)
|
||||
|
||||
if inverted:
|
||||
# compute cost per fold, then aggregate — avoids max-of-mean bias
|
||||
best_val_per_fold = np.nanmax(val_mat, axis=1, keepdims=True) # (n_folds, 1)
|
||||
best_hld_per_fold = np.nanmax(hld_mat, axis=1, keepdims=True)
|
||||
delta_val = best_val_per_fold - val_mat # (n_folds, n_epochs)
|
||||
delta_hld = best_hld_per_fold - hld_mat
|
||||
y_val = np.nanmean(delta_val, axis=0)
|
||||
y_hld = np.nanmean(delta_hld, axis=0)
|
||||
sy_val = np.nanstd(delta_val, axis=0)
|
||||
sy_hld = np.nanstd(delta_hld, axis=0)
|
||||
else:
|
||||
y_val, y_hld = val_mean, hld_mean
|
||||
sy_val, sy_hld = val_std, hld_std
|
||||
|
||||
fig, ax = plt.subplots(figsize=(11, 5))
|
||||
|
||||
if inverted:
|
||||
# faint per-fold lines
|
||||
for i in range(delta_val.shape[0]):
|
||||
ax.plot(epochs, delta_val[i], color="steelblue", linewidth=0.6, alpha=0.18)
|
||||
ax.plot(epochs, delta_hld[i], color="firebrick", linewidth=0.6, alpha=0.18)
|
||||
|
||||
ax.plot(epochs, y_val, color="steelblue", linewidth=2.0,
|
||||
label="Best val − val@N (val cost of stopping early)" if inverted
|
||||
else "Val AUC (best ckpt up to N)")
|
||||
ax.fill_between(epochs, y_val - sy_val, y_val + sy_val, color="steelblue", alpha=0.15)
|
||||
|
||||
ax.plot(epochs, y_hld, color="firebrick", linewidth=2.0,
|
||||
label="Best hld − hld@N (holdout cost of stopping early)" if inverted
|
||||
else "Holdout AUC (at best val ckpt)")
|
||||
ax.fill_between(epochs, y_hld - sy_hld, y_hld + sy_hld, color="firebrick", alpha=0.15)
|
||||
|
||||
if inverted:
|
||||
ax.axhline(0, color="black", linewidth=1.0, linestyle="--", alpha=0.4)
|
||||
|
||||
# CI-crosses-zero regions (with optional tolerance)
|
||||
val_ci_zero = (y_val - sy_val) <= ci_tol
|
||||
hld_ci_zero = (y_hld - sy_hld) <= ci_tol
|
||||
both_ci_zero = val_ci_zero & hld_ci_zero
|
||||
|
||||
ymax = max(np.nanmax(y_val), np.nanmax(y_hld)) * 1.15
|
||||
ax.fill_between(epochs, 0, ymax, where=val_ci_zero,
|
||||
color="steelblue", alpha=0.12, label="val CI ≤ 0")
|
||||
ax.fill_between(epochs, 0, ymax, where=hld_ci_zero,
|
||||
color="firebrick", alpha=0.12, label="holdout CI ≤ 0")
|
||||
ax.fill_between(epochs, 0, ymax, where=both_ci_zero,
|
||||
color="purple", alpha=0.20, label="both CI ≤ 0")
|
||||
|
||||
ax.set_ylabel("AUC lost vs best achievable")
|
||||
ax.set_ylim(-0.05, ymax)
|
||||
legend_loc = "upper right"
|
||||
else:
|
||||
# gap curve on twin axis
|
||||
gap_mean = val_mean - hld_mean
|
||||
ax2 = ax.twinx()
|
||||
ax2.plot(epochs, gap_mean, color="darkorange", linewidth=1.5,
|
||||
linestyle="--", alpha=0.7, label="Val−Holdout gap")
|
||||
ax2.set_ylabel("Val − Holdout gap", color="darkorange", fontsize=9)
|
||||
ax2.tick_params(axis="y", labelcolor="darkorange")
|
||||
ax2.set_ylim(-0.1, 0.4)
|
||||
lines2, labels2 = ax2.get_legend_handles_labels()
|
||||
|
||||
best_hld_ep = epochs[np.nanargmax(hld_mean)]
|
||||
best_hld_val = hld_mean[np.nanargmax(hld_mean)]
|
||||
ax.axvline(best_hld_ep, color="firebrick", linewidth=1.2, linestyle=":",
|
||||
alpha=0.8, label=f"peak holdout @ epoch {best_hld_ep} ({best_hld_val:.3f})")
|
||||
stable = epochs >= 3
|
||||
min_gap_ep = epochs[stable][np.nanargmin(gap_mean[stable])]
|
||||
ax.axvline(min_gap_ep, color="darkorange", linewidth=1.2, linestyle=":",
|
||||
alpha=0.8, label=f"min gap @ epoch {min_gap_ep}")
|
||||
ax.set_ylabel("AUC")
|
||||
ax.set_ylim(0.5, 1.05)
|
||||
legend_loc = "lower right"
|
||||
|
||||
ax.set_xlabel("Stopping epoch N")
|
||||
ax.set_title(title)
|
||||
ax.grid(axis="y", linestyle="--", alpha=0.35)
|
||||
|
||||
lines1, labels1 = ax.get_legend_handles_labels()
|
||||
if not inverted:
|
||||
lines1 += lines2; labels1 += labels2
|
||||
ax.legend(lines1, labels1, fontsize=8, loc=legend_loc)
|
||||
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=150)
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out_path}")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--run-dir", default="analysis_data/pipeline_10x5")
|
||||
ap.add_argument("--eval-mode", default="binary")
|
||||
ap.add_argument("--tower-mode", default="ensemble")
|
||||
ap.add_argument("--head", default="fused", choices=list(HEAD_COL))
|
||||
ap.add_argument("--out", default=None)
|
||||
ap.add_argument("--inverted", action="store_true",
|
||||
help="Plot best-achievable minus current (cost of stopping early)")
|
||||
ap.add_argument("--ci-tol", type=float, default=0.0,
|
||||
help="Tolerance for CI-crosses-zero shading (default 0.0)")
|
||||
args = ap.parse_args()
|
||||
|
||||
run_dir = Path(args.run_dir)
|
||||
val_col, hld_col = HEAD_COL[args.head]
|
||||
|
||||
logs = load_fold_logs(run_dir, args.eval_mode, args.tower_mode, val_col, hld_col)
|
||||
if not logs:
|
||||
print("No epoch_log.csv files found.")
|
||||
return
|
||||
print(f"Loaded {len(logs)} fold logs")
|
||||
|
||||
epochs, val_mat, hld_mat = sweep(logs, val_col, hld_col)
|
||||
|
||||
tol_tag = f"_tol{args.ci_tol}" if args.ci_tol else ""
|
||||
suffix = f"_inverted{tol_tag}" if args.inverted else ""
|
||||
out = Path(args.out) if args.out else (
|
||||
run_dir / args.eval_mode / args.tower_mode / "plots" /
|
||||
f"early_stopping_sweep_{args.head}{suffix}.png"
|
||||
)
|
||||
title = ("Simulated early stopping — cost of stopping at epoch N\n"
|
||||
if args.inverted else
|
||||
"Simulated early stopping sweep\n")
|
||||
title += f"{run_dir.name} · {args.eval_mode}/{args.tower_mode} · head={args.head}"
|
||||
if args.inverted and args.ci_tol:
|
||||
title += f" (CI tol={args.ci_tol})"
|
||||
plot(epochs, val_mat, hld_mat, out, title, inverted=args.inverted, ci_tol=args.ci_tol)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Plot per-epoch holdout metrics across all folds in a 10x5 (or any multi-rep) run.
|
||||
|
||||
Each fold gets its own line. Lines are coloured by rep.
|
||||
|
||||
Modes
|
||||
-----
|
||||
holdout — raw holdout AUC per epoch (original plot)
|
||||
delta — val_auc - holdout_auc per epoch (generalization gap;
|
||||
closer to 0 = val most faithfully reflects holdout)
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/output_analysis/visualizations/plot_holdout_epoch_curves.py \
|
||||
--run-dir analysis_data/pipeline_10x5 \
|
||||
--eval-mode binary \
|
||||
--tower-mode ensemble \
|
||||
--head fused \
|
||||
--mode delta
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
HEAD_COL = {
|
||||
"fused": ("ensemble_val_auc", "ensemble_holdout_auc"),
|
||||
"img": ("ensemble_val_auc_img", "ensemble_holdout_auc_img"),
|
||||
"md": ("ensemble_val_auc_md", "ensemble_holdout_auc_md"),
|
||||
"classic": ("classic_val_auc", "classic_holdout_auc"),
|
||||
}
|
||||
|
||||
|
||||
def load_curves(run_dir: Path, eval_mode: str, tower_mode: str,
|
||||
val_col: str, hld_col: str, mode: str):
|
||||
"""
|
||||
Returns list of (rep, fold, epochs_array, values_array).
|
||||
mode='holdout' → values = holdout_auc
|
||||
mode='delta' → values = val_auc - holdout_auc
|
||||
"""
|
||||
curves = []
|
||||
for rep_dir in sorted(run_dir.glob("rep*")):
|
||||
mode_dir = rep_dir / eval_mode / tower_mode
|
||||
if not mode_dir.exists():
|
||||
continue
|
||||
fold_dirs = sorted(
|
||||
[d for d in mode_dir.iterdir() if d.is_dir() and d.name.startswith("fold")],
|
||||
key=lambda p: int(p.name.replace("fold", "")),
|
||||
)
|
||||
for fd in fold_dirs:
|
||||
log = fd / "epoch_log.csv"
|
||||
if not log.exists():
|
||||
continue
|
||||
df = pd.read_csv(log)
|
||||
needed = [hld_col] if mode == "holdout" else [val_col, hld_col]
|
||||
if any(c not in df.columns for c in needed):
|
||||
continue
|
||||
df = df.dropna(subset=needed)
|
||||
if mode == "holdout":
|
||||
values = df[hld_col].to_numpy()
|
||||
elif mode == "delta":
|
||||
values = (df[val_col] - df[hld_col]).to_numpy()
|
||||
else: # delta_adj
|
||||
val_arr = df[val_col].to_numpy()
|
||||
hld_arr = df[hld_col].to_numpy()
|
||||
val_best = np.nanmax(val_arr)
|
||||
penalty = val_best - val_arr # 0 when val is at its peak
|
||||
values = (val_arr - hld_arr) + penalty
|
||||
curves.append((rep_dir.name, fd.name, df["epoch"].to_numpy(), values))
|
||||
return curves
|
||||
|
||||
|
||||
def _build_mean_matrix(curves):
|
||||
all_ep = max(len(e) for _, _, e, _ in curves)
|
||||
mat = np.full((len(curves), all_ep), np.nan)
|
||||
for i, (_, _, e, a) in enumerate(curves):
|
||||
mat[i, :len(a)] = a
|
||||
return mat, np.arange(1, all_ep + 1)
|
||||
|
||||
|
||||
def plot(curves, mode: str, out_path: Path, title: str):
|
||||
reps = sorted(set(r for r, _, _, _ in curves))
|
||||
cmap = matplotlib.colormaps.get_cmap("tab10")
|
||||
rep_color = {r: cmap(i / max(len(reps) - 1, 1)) for i, r in enumerate(reps)}
|
||||
|
||||
fig, ax = plt.subplots(figsize=(12, 6))
|
||||
|
||||
for rep, fold, epochs, vals in curves:
|
||||
ax.plot(epochs, vals, color=rep_color[rep], alpha=0.3, linewidth=0.9)
|
||||
|
||||
# per-rep mean
|
||||
for rep in reps:
|
||||
rep_curves = [(e, a) for r, _, e, a in curves if r == rep]
|
||||
max_ep = max(len(e) for e, _ in rep_curves)
|
||||
mat = np.full((len(rep_curves), max_ep), np.nan)
|
||||
for i, (e, a) in enumerate(rep_curves):
|
||||
mat[i, :len(a)] = a
|
||||
mean_curve = np.nanmean(mat, axis=0)
|
||||
ax.plot(np.arange(1, max_ep + 1), mean_curve,
|
||||
color=rep_color[rep], linewidth=1.8, alpha=0.85, label=rep)
|
||||
|
||||
# global mean ± std
|
||||
all_mat, ep_axis = _build_mean_matrix(curves)
|
||||
global_mean = np.nanmean(all_mat, axis=0)
|
||||
global_std = np.nanstd(all_mat, axis=0)
|
||||
ax.plot(ep_axis, global_mean, color="black", linewidth=2.5, zorder=5, label="global mean")
|
||||
ax.fill_between(ep_axis, global_mean - global_std, global_mean + global_std,
|
||||
color="black", alpha=0.12, zorder=4)
|
||||
|
||||
if mode in ("delta", "delta_adj"):
|
||||
ax.axhline(0, color="black", linewidth=1.0, linestyle="--", alpha=0.5)
|
||||
if mode == "delta":
|
||||
ax.set_ylabel("Val AUC − Holdout AUC (gap)")
|
||||
else:
|
||||
ax.set_ylabel("(Val − Holdout) + (ValBest − Val) (adjusted gap)")
|
||||
min_ep = int(ep_axis[np.nanargmin(global_mean)])
|
||||
min_val = global_mean[np.nanargmin(global_mean)]
|
||||
ax.axvline(min_ep, color="red", linewidth=1.2, linestyle=":", alpha=0.7,
|
||||
label=f"min adjusted gap @ epoch {min_ep} ({min_val:+.3f})")
|
||||
else:
|
||||
ax.set_ylabel("Holdout AUC")
|
||||
ax.set_ylim(0, 1.05)
|
||||
|
||||
ax.set_xlabel("Epoch")
|
||||
ax.set_title(title)
|
||||
ax.legend(fontsize=7, ncol=2, loc="upper right" if mode == "delta" else "lower right")
|
||||
ax.grid(axis="y", linestyle="--", alpha=0.4)
|
||||
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=150)
|
||||
plt.close(fig)
|
||||
print(f"Saved → {out_path}")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--run-dir", default="analysis_data/pipeline_10x5")
|
||||
ap.add_argument("--eval-mode", default="binary")
|
||||
ap.add_argument("--tower-mode", default="ensemble")
|
||||
ap.add_argument("--head", default="fused", choices=list(HEAD_COL))
|
||||
ap.add_argument("--mode", default="holdout", choices=["holdout", "delta", "delta_adj"])
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
run_dir = Path(args.run_dir)
|
||||
val_col, hld_col = HEAD_COL[args.head]
|
||||
curves = load_curves(run_dir, args.eval_mode, args.tower_mode,
|
||||
val_col, hld_col, args.mode)
|
||||
|
||||
if not curves:
|
||||
print("No epoch_log.csv files found — check --run-dir / --eval-mode / --tower-mode")
|
||||
return
|
||||
|
||||
print(f"Loaded {len(curves)} fold curves, up to {max(len(e) for _,_,e,_ in curves)} epochs each")
|
||||
|
||||
out = Path(args.out) if args.out else (
|
||||
run_dir / args.eval_mode / args.tower_mode / "plots" /
|
||||
f"holdout_epoch_curves_{args.head}_{args.mode}.png"
|
||||
)
|
||||
label = {"holdout": "holdout AUC", "delta": "val−holdout gap", "delta_adj": "val−holdout gap (val-adjusted)"}[args.mode]
|
||||
title = (f"Per-fold {label} by epoch\n"
|
||||
f"{run_dir.name} · {args.eval_mode}/{args.tower_mode} · head={args.head}")
|
||||
plot(curves, args.mode, out, title)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Per-epoch learning curves for mdonly runs.
|
||||
|
||||
Reads epoch_log.csv from each fold dir and plots val_auc, val_acc,
|
||||
hld_auc, hld_acc — one figure per metric, all folds as individual lines.
|
||||
|
||||
Usage:
|
||||
python scripts/output_analysis/visualizations/plot_mdonly_curves.py \
|
||||
--run-dirs analysis_data/pipeline_mdonly_50ep \
|
||||
analysis_data/pipeline_mdonly_200ep \
|
||||
analysis_data/pipeline_mdonly_500ep \
|
||||
--eval-mode binary
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
METRICS = [
|
||||
("val_auc", "Val AUC"),
|
||||
("val_acc", "Val Accuracy"),
|
||||
("hld_auc", "Holdout AUC"),
|
||||
("hld_acc", "Holdout Accuracy"),
|
||||
]
|
||||
|
||||
PHASE_SHADING = {
|
||||
"tower_warmup": "#d0e8ff",
|
||||
"fused_warmup": "#d0ffe8",
|
||||
}
|
||||
|
||||
|
||||
def _load_folds(mode_dir: Path) -> list[pd.DataFrame]:
|
||||
fold_dirs = sorted(
|
||||
[p for p in mode_dir.glob("fold*") if p.is_dir()],
|
||||
key=lambda p: int(p.name.replace("fold", "")),
|
||||
)
|
||||
frames = []
|
||||
for fd in fold_dirs:
|
||||
csv = fd / "epoch_log.csv"
|
||||
if not csv.exists():
|
||||
print(f" [warn] {csv} not found, skipping")
|
||||
continue
|
||||
df = pd.read_csv(csv)
|
||||
df["_fold"] = int(fd.name.replace("fold", ""))
|
||||
frames.append(df)
|
||||
return frames
|
||||
|
||||
|
||||
def _shade_warmup(ax: plt.Axes, df: pd.DataFrame) -> None:
|
||||
"""Shade warmup phase regions based on first fold's phase column."""
|
||||
if "phase" not in df.columns:
|
||||
return
|
||||
prev_phase = None
|
||||
start = None
|
||||
for _, row in df.iterrows():
|
||||
phase = row["phase"]
|
||||
ep = row["epoch"]
|
||||
if phase != prev_phase:
|
||||
if prev_phase in PHASE_SHADING and start is not None:
|
||||
ax.axvspan(start - 0.5, ep - 0.5, color=PHASE_SHADING[prev_phase],
|
||||
alpha=0.35, zorder=0, label=f"{prev_phase.replace('_', ' ')}")
|
||||
start = ep
|
||||
prev_phase = phase
|
||||
# close last span
|
||||
if prev_phase in PHASE_SHADING and start is not None:
|
||||
ax.axvspan(start - 0.5, df["epoch"].max() + 0.5,
|
||||
color=PHASE_SHADING[prev_phase], alpha=0.35, zorder=0)
|
||||
|
||||
|
||||
def plot_curves(
|
||||
run_dirs: list[Path],
|
||||
eval_mode: str,
|
||||
tower_mode: str,
|
||||
out_dir: Path | None,
|
||||
) -> None:
|
||||
# Collect (label, frames) pairs
|
||||
datasets: list[tuple[str, list[pd.DataFrame]]] = []
|
||||
for rd in run_dirs:
|
||||
mode_dir = rd / eval_mode / tower_mode
|
||||
if not mode_dir.exists():
|
||||
print(f" [skip] {mode_dir} not found")
|
||||
continue
|
||||
frames = _load_folds(mode_dir)
|
||||
if not frames:
|
||||
print(f" [skip] no epoch_log.csv found under {mode_dir}")
|
||||
continue
|
||||
datasets.append((rd.name, frames))
|
||||
|
||||
if not datasets:
|
||||
print("No data found — nothing to plot.")
|
||||
return
|
||||
|
||||
# One figure per metric
|
||||
for metric_key, metric_label in METRICS:
|
||||
# Check any fold actually has this metric with non-nan values
|
||||
has_data = any(
|
||||
not frames[0][metric_key].isna().all()
|
||||
for _, frames in datasets
|
||||
if frames and metric_key in frames[0].columns
|
||||
)
|
||||
if not has_data:
|
||||
continue
|
||||
|
||||
n_runs = len(datasets)
|
||||
fig, axes = plt.subplots(1, n_runs, figsize=(5 * n_runs, 4.5), squeeze=False)
|
||||
|
||||
for col_idx, (run_label, frames) in enumerate(datasets):
|
||||
ax = axes[0][col_idx]
|
||||
if frames and metric_key in frames[0].columns:
|
||||
_shade_warmup(ax, frames[0])
|
||||
|
||||
colours = plt.cm.tab10(np.linspace(0, 0.9, len(frames)))
|
||||
for frame, colour in zip(frames, colours):
|
||||
if metric_key not in frame.columns:
|
||||
continue
|
||||
vals = frame[metric_key].values
|
||||
epochs = frame["epoch"].values
|
||||
mask = ~np.isnan(vals.astype(float))
|
||||
if mask.sum() == 0:
|
||||
continue
|
||||
ax.plot(epochs[mask], vals[mask],
|
||||
linewidth=1.4, color=colour,
|
||||
label=f"fold {frame['_fold'].iloc[0]}")
|
||||
|
||||
ax.set_title(run_label, fontsize=10)
|
||||
ax.set_xlabel("Epoch")
|
||||
if col_idx == 0:
|
||||
ax.set_ylabel(metric_label)
|
||||
ax.legend(fontsize=7, loc="lower right")
|
||||
ax.grid(True, linewidth=0.4, alpha=0.5)
|
||||
|
||||
fig.suptitle(f"{metric_label} [{eval_mode} / {tower_mode}]", fontsize=12)
|
||||
fig.tight_layout()
|
||||
|
||||
dest = out_dir or (run_dirs[0].parent / "mdonly_plots")
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
fname = f"mdonly_{metric_key}_{eval_mode}_{tower_mode}.png"
|
||||
fig.savefig(dest / fname, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {dest / fname}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--run-dirs", nargs="+", required=True,
|
||||
help="One or more run directories (e.g. analysis_data/pipeline_mdonly_50ep).")
|
||||
ap.add_argument("--eval-mode", default="binary", choices=["binary", "multiclass"])
|
||||
ap.add_argument("--tower-mode", default="single", choices=["single", "ensemble"])
|
||||
ap.add_argument("--out", default=None,
|
||||
help="Output directory for plots (default: {first_run_dir}/../mdonly_plots).")
|
||||
args = ap.parse_args()
|
||||
|
||||
run_dirs = [Path(d) for d in args.run_dirs]
|
||||
out_dir = Path(args.out) if args.out else None
|
||||
plot_curves(run_dirs, args.eval_mode, args.tower_mode, out_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -80,22 +80,74 @@ def _fold_colours(n_folds: int) -> dict[int, dict[int, tuple]]:
|
||||
return out
|
||||
|
||||
|
||||
def _load_folds(run_dir: Path, head: str) -> list[pd.DataFrame]:
|
||||
fold_dirs = sorted(
|
||||
def _fold_dirs(run_dir: Path) -> list[Path]:
|
||||
return sorted(
|
||||
[p for p in run_dir.glob("fold*") if p.is_dir() and re.search(r"\d+", p.name)],
|
||||
key=lambda p: int(re.search(r"\d+", p.name).group()),
|
||||
)
|
||||
if not fold_dirs:
|
||||
|
||||
|
||||
def _detect_heads(run_dir: Path) -> list[str]:
|
||||
"""Return all head names available in the first non-empty fold dir."""
|
||||
for fd in _fold_dirs(run_dir):
|
||||
found: list[str] = []
|
||||
# Standard heads come from the predictions CSV — prefer per-eye
|
||||
csv = fd / "predictions_pereye.csv"
|
||||
if not csv.exists():
|
||||
csv = fd / "predictions_classic.csv"
|
||||
if not csv.exists():
|
||||
csv = fd / "predictions.csv"
|
||||
if csv.exists():
|
||||
cols = pd.read_csv(csv, nrows=0).columns.tolist()
|
||||
for h in ["fused", "img", "md"]:
|
||||
if f"prob_{h}_c0" in cols:
|
||||
found.append(h)
|
||||
# Fusion head lives in a separate npy
|
||||
if (fd / "probs_fused_head.npy").exists():
|
||||
found.append("fused_head")
|
||||
if found:
|
||||
return found
|
||||
return ["fused"] # safe fallback
|
||||
|
||||
|
||||
def _load_folds(run_dir: Path, head: str) -> list[pd.DataFrame]:
|
||||
dirs = _fold_dirs(run_dir)
|
||||
if not dirs:
|
||||
raise FileNotFoundError(f"No fold* directories found under {run_dir}")
|
||||
frames = []
|
||||
for fd in fold_dirs:
|
||||
csv = fd / "predictions_classic.csv"
|
||||
for fd in dirs:
|
||||
fold_num = int(re.search(r"\d+", fd.name).group())
|
||||
# fused_head is stored as npy, not in the predictions CSV
|
||||
if head == "fused_head":
|
||||
y_path = fd / "y_true.npy"
|
||||
p_path = fd / "probs_fused_head.npy"
|
||||
if not y_path.exists() or not p_path.exists():
|
||||
print(f" [warn] fused_head npy not found in {fd}, skipping")
|
||||
continue
|
||||
y = np.load(y_path)
|
||||
p = np.load(p_path)
|
||||
df = pd.DataFrame({"y_true": y})
|
||||
for c in range(p.shape[1]):
|
||||
df[f"prob_fused_head_c{c}"] = p[:, c]
|
||||
df["_fold"] = fold_num
|
||||
frames.append(df)
|
||||
continue
|
||||
# Standard heads from predictions CSV — prefer per-eye (2× dots, no OD/OS averaging)
|
||||
csv = fd / "predictions_pereye.csv"
|
||||
if not csv.exists():
|
||||
print(f" [warn] {csv} not found, skipping")
|
||||
csv = fd / "predictions_classic.csv"
|
||||
if not csv.exists():
|
||||
csv = fd / "predictions.csv"
|
||||
if not csv.exists():
|
||||
print(f" [warn] no predictions CSV found in {fd}, skipping")
|
||||
continue
|
||||
df = pd.read_csv(csv)
|
||||
df["_fold"] = int(re.search(r"\d+", fd.name).group())
|
||||
df["_fold"] = fold_num
|
||||
frames.append(df)
|
||||
if not frames:
|
||||
raise FileNotFoundError(
|
||||
f"No data found for head='{head}' in any fold dir under {run_dir}"
|
||||
)
|
||||
return frames
|
||||
|
||||
|
||||
@@ -627,7 +679,9 @@ def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--run-dir", required=True)
|
||||
ap.add_argument("--head", default="fused", choices=["fused", "img", "md"])
|
||||
ap.add_argument("--head", default=None,
|
||||
choices=["fused", "fused_head", "img", "md"],
|
||||
help="Head to plot. Omit to auto-detect and plot all available heads.")
|
||||
ap.add_argument("--style", default="strips",
|
||||
choices=["strips", "confidence", "triangle", "triangle3d"],
|
||||
help="strips: X=true class, Y=P(Glaucoma). "
|
||||
@@ -638,14 +692,22 @@ def main():
|
||||
|
||||
rd = Path(args.run_dir)
|
||||
od = Path(args.out) if args.out else None
|
||||
if args.style == "confidence":
|
||||
plot_confidence(rd, head=args.head, out_dir=od)
|
||||
elif args.style == "triangle":
|
||||
plot_triangle(rd, head=args.head, out_dir=od)
|
||||
elif args.style == "triangle3d":
|
||||
plot_triangle_3d(rd, head=args.head, out_dir=od)
|
||||
else:
|
||||
plot_strip(rd, head=args.head, out_dir=od)
|
||||
|
||||
heads = [args.head] if args.head else _detect_heads(rd)
|
||||
print(f"Heads to plot: {heads}")
|
||||
|
||||
plot_fn = {
|
||||
"confidence": plot_confidence,
|
||||
"triangle": plot_triangle,
|
||||
"triangle3d": plot_triangle_3d,
|
||||
}.get(args.style, plot_strip)
|
||||
|
||||
for head in heads:
|
||||
print(f"\n--- {head} ---")
|
||||
try:
|
||||
plot_fn(rd, head=head, out_dir=od)
|
||||
except Exception as exc:
|
||||
print(f" [skip] {head}: {exc}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -57,12 +57,10 @@ _PROBS_PRIORITY: dict[str, list[str]] = {
|
||||
_ALL_PROBS = ["probs_fused_head", "probs_fused", "probs_bilat", "probs_classic"]
|
||||
|
||||
|
||||
def detect_probs_stem(fold_dir: Path, tower_mode: str | None) -> str | None:
|
||||
def detect_probs_stems(fold_dir: Path, tower_mode: str | None) -> list[str]:
|
||||
"""Return all present probs stems (in priority order) for this fold dir."""
|
||||
priority = _PROBS_PRIORITY.get(tower_mode, _ALL_PROBS) if tower_mode else _ALL_PROBS
|
||||
for stem in priority:
|
||||
if (fold_dir / f"{stem}.npy").exists():
|
||||
return stem
|
||||
return None
|
||||
return [stem for stem in priority if (fold_dir / f"{stem}.npy").exists()]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -246,39 +244,44 @@ def main() -> None:
|
||||
if not fold_dirs:
|
||||
raise SystemExit(f"No fold subdirectories found in {mode_dir}")
|
||||
|
||||
# Determine probs stem
|
||||
probs_stem = args.probs
|
||||
if probs_stem is None:
|
||||
# Determine probs stems to plot
|
||||
if args.probs is not None:
|
||||
stems_to_plot = [args.probs]
|
||||
else:
|
||||
# Collect all stems present across any fold dir
|
||||
seen: list[str] = []
|
||||
for fd in fold_dirs:
|
||||
probs_stem = detect_probs_stem(fd, args.tower_mode)
|
||||
if probs_stem:
|
||||
break
|
||||
if probs_stem is None:
|
||||
raise SystemExit(f"Could not detect a probs file in {mode_dir}/fold*/")
|
||||
print(f"Using probs: {probs_stem}.npy")
|
||||
|
||||
# Load all folds
|
||||
per_fold: list[tuple[int, dict]] = []
|
||||
for fd in fold_dirs:
|
||||
fold_idx = int(fd.name.replace("fold", ""))
|
||||
result = load_fold(fd, probs_stem, args.eval_mode)
|
||||
if result is None:
|
||||
print(f" [skip] fold {fold_idx}: missing y_true or {probs_stem}.npy")
|
||||
continue
|
||||
y, p = result
|
||||
curves = per_class_roc(y, p)
|
||||
per_fold.append((fold_idx, curves))
|
||||
auc_str = " ".join(
|
||||
f"class{k}={v['auc']:.3f}" for k, v in curves.items()
|
||||
)
|
||||
print(f" fold {fold_idx}: {auc_str}")
|
||||
|
||||
if not per_fold:
|
||||
raise SystemExit("No usable folds — nothing to plot.")
|
||||
for s in detect_probs_stems(fd, args.tower_mode):
|
||||
if s not in seen:
|
||||
seen.append(s)
|
||||
stems_to_plot = seen
|
||||
if not stems_to_plot:
|
||||
raise SystemExit(f"Could not detect any probs file in {mode_dir}/fold*/")
|
||||
print(f"Probs stems to plot: {stems_to_plot}")
|
||||
|
||||
out_dir = mode_dir / "plots"
|
||||
plot_perfold(per_fold, out_dir, args.class_names, probs_stem, args.eval_mode)
|
||||
plot_mean_ovr(per_fold, out_dir, args.class_names, probs_stem, args.eval_mode)
|
||||
for probs_stem in stems_to_plot:
|
||||
print(f"\n--- {probs_stem} ---")
|
||||
per_fold: list[tuple[int, dict]] = []
|
||||
for fd in fold_dirs:
|
||||
fold_idx = int(fd.name.replace("fold", ""))
|
||||
result = load_fold(fd, probs_stem, args.eval_mode)
|
||||
if result is None:
|
||||
print(f" [skip] fold {fold_idx}: missing y_true or {probs_stem}.npy")
|
||||
continue
|
||||
y, p = result
|
||||
curves = per_class_roc(y, p)
|
||||
per_fold.append((fold_idx, curves))
|
||||
auc_str = " ".join(f"class{k}={v['auc']:.3f}" for k, v in curves.items())
|
||||
print(f" fold {fold_idx}: {auc_str}")
|
||||
|
||||
if not per_fold:
|
||||
print(f" No usable folds for {probs_stem}, skipping.")
|
||||
continue
|
||||
|
||||
plot_perfold(per_fold, out_dir, args.class_names, probs_stem, args.eval_mode)
|
||||
plot_mean_ovr(per_fold, out_dir, args.class_names, probs_stem, args.eval_mode)
|
||||
|
||||
print(f"\nPlots written to {out_dir}")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user