Add new scripts and configurations for model comparison and analysis
- Introduced `poster_model_comparison.py` for generating model comparison figures. - Added `plot_poster_roc_comparison.py` for creating ROC comparison figures for PAPILA binary classification. - Created new JSON configuration files for clinical solo models with and without geometry injection. - Implemented batch dispatch updates in `batch_dispatch.py` to utilize run names from configurations. - Added analysis scripts: `compare_grid.py`, `inspect_embeddings.py`, and `summarize_run.py` for evaluating model performance and feature embeddings. - Created experiment configurations for various training scenarios, including warm sweeps and promoting successful runs. - Added binary ROC comparison and model comparison figures to the results directory.
This commit is contained in:
rpotter6298
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Poster-facing model comparison figure in the style of head_comparison.png.
|
||||
|
||||
Compares:
|
||||
- MD Only
|
||||
- Image Only
|
||||
- Ensemble Fusion
|
||||
|
||||
using the same run sources as the poster ROC comparison script.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
from v3.scripts.output_analysis.plot_poster_roc_comparison import (
|
||||
DEFAULT_CURVES,
|
||||
REPO_ROOT,
|
||||
build_curve_summary,
|
||||
load_curve_predictions,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_OUT = REPO_ROOT / "v3" / "figures" / "explainability" / "poster_model_comparison.png"
|
||||
MODEL_ORDER = ["Clinical Data Only", "Image Only", "Ensemble Fusion"]
|
||||
|
||||
C_NORMAL = "#4c72b0"
|
||||
C_GLAUCOMA = "#c44e52"
|
||||
BG = "#e8e8e8"
|
||||
|
||||
|
||||
def _select_specs():
|
||||
selected = []
|
||||
for label in MODEL_ORDER:
|
||||
matches = [spec for spec in DEFAULT_CURVES if spec.label == label]
|
||||
if not matches:
|
||||
raise ValueError(f"Could not find default poster curve for '{label}'")
|
||||
selected.append(matches[0])
|
||||
return selected
|
||||
|
||||
|
||||
def make_model_comparison(out_path: Path) -> None:
|
||||
specs = _select_specs()
|
||||
rng = np.random.default_rng(42)
|
||||
|
||||
fig, axes = plt.subplots(1, len(specs), figsize=(11, 4.5), sharey=True)
|
||||
if len(specs) == 1:
|
||||
axes = [axes]
|
||||
fig.patch.set_facecolor(BG)
|
||||
fig.suptitle(
|
||||
"Model Comparison — P(Glaucoma) by True Class",
|
||||
fontsize=12,
|
||||
fontweight="bold",
|
||||
)
|
||||
|
||||
for ax, spec in zip(axes, specs):
|
||||
df, score_col = load_curve_predictions(spec)
|
||||
summary = build_curve_summary(df, score_col)
|
||||
|
||||
ax.set_facecolor(BG)
|
||||
data_by_class = [df.loc[df["y_true"] == cls, score_col].values for cls in [0, 1]]
|
||||
|
||||
vp = ax.violinplot(
|
||||
data_by_class,
|
||||
positions=[0, 1],
|
||||
widths=0.6,
|
||||
showmedians=True,
|
||||
showextrema=False,
|
||||
)
|
||||
for body, color in zip(vp["bodies"], [C_NORMAL, C_GLAUCOMA]):
|
||||
body.set_facecolor(color)
|
||||
body.set_alpha(0.35)
|
||||
vp["cmedians"].set_color("#222")
|
||||
vp["cmedians"].set_linewidth(2)
|
||||
|
||||
for cls, color in zip([0, 1], [C_NORMAL, C_GLAUCOMA]):
|
||||
vals = data_by_class[cls]
|
||||
jitter = rng.uniform(-0.12, 0.12, len(vals))
|
||||
ax.scatter(
|
||||
cls + jitter,
|
||||
vals,
|
||||
color=color,
|
||||
s=4,
|
||||
alpha=0.30,
|
||||
linewidths=0,
|
||||
zorder=3,
|
||||
)
|
||||
|
||||
ax.axhline(0.5, color="#888", lw=1.0, ls="--", alpha=0.6)
|
||||
ax.set_xticks([0, 1])
|
||||
ax.set_xticklabels(["Normal", "Glaucoma"], fontsize=9)
|
||||
ax.set_title(spec.label, fontsize=10, fontweight="bold", color=spec.color)
|
||||
ax.set_ylim(-0.05, 1.05)
|
||||
ax.grid(axis="y", alpha=0.3)
|
||||
if ax is axes[0]:
|
||||
ax.set_ylabel("Predicted P(Glaucoma)", fontsize=10)
|
||||
|
||||
ax.text(
|
||||
0.97,
|
||||
0.04,
|
||||
f"AUC = {summary['auc_mean']:.3f} ± {summary['auc_std']:.3f}",
|
||||
transform=ax.transAxes,
|
||||
ha="right",
|
||||
va="bottom",
|
||||
fontsize=9,
|
||||
color="#333",
|
||||
bbox=dict(facecolor="white", alpha=0.65, edgecolor="none", pad=2),
|
||||
)
|
||||
|
||||
fig.tight_layout()
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out_path, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"Saved: {out_path}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--out", type=Path, default=DEFAULT_OUT)
|
||||
args = ap.parse_args()
|
||||
make_model_comparison(args.out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,283 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Poster-facing ROC comparison for PAPILA binary glaucoma classification.
|
||||
|
||||
Builds one combined ROC figure with four curves:
|
||||
- MD Only
|
||||
- Image Only
|
||||
- Single Fusion
|
||||
- Ensemble Fusion
|
||||
|
||||
The default setup mixes two artifact layouts:
|
||||
1. Dedicated unimodal runs saved under analysis_data/.../fold*/{y_true,probs}.npy
|
||||
2. Repeated CV fusion runs saved under v3/results/.../rep*/.../predictions_test.csv
|
||||
|
||||
Update DEFAULT_CURVES below if you want different source runs or labels.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.metrics import roc_auc_score, roc_curve
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
RESULTS_ROOT = REPO_ROOT / "v3" / "results"
|
||||
DEFAULT_OUT = REPO_ROOT / "results" / "poster" / "papila_binary_roc_comparison.png"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CurveSpec:
|
||||
label: str
|
||||
source_kind: str
|
||||
color: str
|
||||
mode_dir: str | None = None
|
||||
probs_stem: str | None = None
|
||||
run: str | None = None
|
||||
tower_path: str | None = None
|
||||
score_col: str | None = None
|
||||
|
||||
|
||||
DEFAULT_CURVES = [
|
||||
CurveSpec(
|
||||
label="Clinical Data Only",
|
||||
source_kind="legacy_npy",
|
||||
color="#4c72b0",
|
||||
mode_dir="analysis_data/pipeline_mdonly_500ep/binary/single",
|
||||
probs_stem="probs_classic",
|
||||
),
|
||||
CurveSpec(
|
||||
label="Image Only",
|
||||
source_kind="v3_csv",
|
||||
color="#dd8452",
|
||||
run="phase2/imageonly_resnet50_proper",
|
||||
tower_path="binary/single",
|
||||
score_col="prob_img_c1",
|
||||
),
|
||||
CurveSpec(
|
||||
label="Single Fusion",
|
||||
source_kind="v3_csv",
|
||||
color="#55a868",
|
||||
run="phase5/single_fused",
|
||||
tower_path="binary/single",
|
||||
score_col="prob_fused_c1",
|
||||
),
|
||||
CurveSpec(
|
||||
label="Ensemble Fusion",
|
||||
source_kind="v3_csv",
|
||||
color="#c44e52",
|
||||
run="phase5/logit_mlp_head",
|
||||
tower_path="binary/ensemble",
|
||||
score_col="prob_fused_c1",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def load_v3_csv_predictions(run: str, tower_path: str, score_col: str) -> pd.DataFrame:
|
||||
run_dir = RESULTS_ROOT / run
|
||||
rows: list[pd.DataFrame] = []
|
||||
for rep_dir in sorted(run_dir.glob("rep*")):
|
||||
mode_dir = rep_dir / tower_path
|
||||
if not mode_dir.exists():
|
||||
continue
|
||||
for fold_dir in sorted(mode_dir.glob("fold[0-9]")):
|
||||
csv_path = fold_dir / "predictions_test.csv"
|
||||
if not csv_path.exists():
|
||||
continue
|
||||
df = pd.read_csv(csv_path, usecols=["y_true", score_col])
|
||||
df["rep"] = rep_dir.name
|
||||
df["fold"] = fold_dir.name
|
||||
rows.append(df)
|
||||
if not rows:
|
||||
raise FileNotFoundError(f"No predictions found under {run_dir}/{tower_path}")
|
||||
return pd.concat(rows, ignore_index=True)
|
||||
|
||||
|
||||
def load_legacy_npy_predictions(mode_dir: str, probs_stem: str) -> pd.DataFrame:
|
||||
root = REPO_ROOT / mode_dir
|
||||
rows: list[pd.DataFrame] = []
|
||||
for fold_dir in sorted(root.glob("fold[0-9]")):
|
||||
y_path = fold_dir / "y_true.npy"
|
||||
p_path = fold_dir / f"{probs_stem}.npy"
|
||||
if not y_path.exists() or not p_path.exists():
|
||||
continue
|
||||
y_true = np.load(y_path)
|
||||
probs = np.load(p_path)
|
||||
if probs.ndim == 2:
|
||||
if probs.shape[1] < 2:
|
||||
raise ValueError(f"Expected 2-class probs in {p_path}")
|
||||
scores = probs[:, 1]
|
||||
else:
|
||||
scores = probs
|
||||
rows.append(
|
||||
pd.DataFrame(
|
||||
{
|
||||
"y_true": y_true,
|
||||
"score": scores,
|
||||
"rep": "rep00",
|
||||
"fold": fold_dir.name,
|
||||
}
|
||||
)
|
||||
)
|
||||
if not rows:
|
||||
raise FileNotFoundError(f"No fold artifacts found under {root}")
|
||||
return pd.concat(rows, ignore_index=True)
|
||||
|
||||
|
||||
def load_curve_predictions(spec: CurveSpec) -> tuple[pd.DataFrame, str]:
|
||||
if spec.source_kind == "v3_csv":
|
||||
if spec.run is None or spec.tower_path is None or spec.score_col is None:
|
||||
raise ValueError(f"Incomplete v3_csv spec: {spec}")
|
||||
return load_v3_csv_predictions(spec.run, spec.tower_path, spec.score_col), spec.score_col
|
||||
if spec.source_kind == "legacy_npy":
|
||||
if spec.mode_dir is None or spec.probs_stem is None:
|
||||
raise ValueError(f"Incomplete legacy_npy spec: {spec}")
|
||||
return load_legacy_npy_predictions(spec.mode_dir, spec.probs_stem), "score"
|
||||
raise ValueError(f"Unknown source_kind: {spec.source_kind}")
|
||||
|
||||
|
||||
def build_curve_summary(df: pd.DataFrame, score_col: str) -> dict[str, object]:
|
||||
if score_col not in df.columns:
|
||||
raise KeyError(f"Missing score column '{score_col}'")
|
||||
|
||||
mean_fpr = np.linspace(0.0, 1.0, 501)
|
||||
tprs: list[np.ndarray] = []
|
||||
aucs: list[float] = []
|
||||
fold_count = 0
|
||||
|
||||
for (_, _), fold_df in df.groupby(["rep", "fold"], sort=True):
|
||||
y_true = fold_df["y_true"].to_numpy()
|
||||
scores = fold_df[score_col].to_numpy()
|
||||
if len(np.unique(y_true)) < 2:
|
||||
continue
|
||||
fpr, tpr, _ = roc_curve(y_true, scores)
|
||||
interp_tpr = np.interp(mean_fpr, fpr, tpr)
|
||||
interp_tpr[0] = 0.0
|
||||
interp_tpr[-1] = 1.0
|
||||
tprs.append(interp_tpr)
|
||||
aucs.append(float(roc_auc_score(y_true, scores)))
|
||||
fold_count += 1
|
||||
|
||||
if not tprs:
|
||||
raise ValueError(f"No valid binary folds found for '{score_col}'")
|
||||
|
||||
tpr_arr = np.vstack(tprs)
|
||||
y_all = df["y_true"].to_numpy()
|
||||
s_all = df[score_col].to_numpy()
|
||||
|
||||
return {
|
||||
"fpr": mean_fpr,
|
||||
"tpr_mean": tpr_arr.mean(axis=0),
|
||||
"tpr_std": tpr_arr.std(axis=0),
|
||||
"auc_mean": float(np.mean(aucs)),
|
||||
"auc_std": float(np.std(aucs)),
|
||||
"auc_pooled": float(roc_auc_score(y_all, s_all)),
|
||||
"fold_count": fold_count,
|
||||
"n_total": int(len(df)),
|
||||
}
|
||||
|
||||
|
||||
def plot_curves(curve_summaries: list[tuple[CurveSpec, dict[str, object]]], out_path: Path) -> None:
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
fig, ax = plt.subplots(figsize=(8.8, 7.0))
|
||||
ax.plot([0, 1], [0, 1], linestyle="--", linewidth=1, color="0.5", alpha=0.8)
|
||||
|
||||
for spec, summary in curve_summaries:
|
||||
fpr = np.asarray(summary["fpr"])
|
||||
tpr_mean = np.asarray(summary["tpr_mean"])
|
||||
tpr_std = np.asarray(summary["tpr_std"])
|
||||
auc_mean = float(summary["auc_mean"])
|
||||
auc_std = float(summary["auc_std"])
|
||||
|
||||
ax.plot(
|
||||
fpr,
|
||||
tpr_mean,
|
||||
color=spec.color,
|
||||
linewidth=2.4,
|
||||
label=f"{spec.label} (AUC {auc_mean:.3f} ± {auc_std:.3f})",
|
||||
)
|
||||
ax.fill_between(
|
||||
fpr,
|
||||
np.clip(tpr_mean - tpr_std, 0, 1),
|
||||
np.clip(tpr_mean + tpr_std, 0, 1),
|
||||
color=spec.color,
|
||||
alpha=0.12,
|
||||
)
|
||||
|
||||
ax.set_xlim(-0.01, 1.01)
|
||||
ax.set_ylim(-0.01, 1.01)
|
||||
ax.set_xlabel("False Positive Rate")
|
||||
ax.set_ylabel("True Positive Rate")
|
||||
ax.set_title("PAPILA Binary ROC Comparison")
|
||||
ax.grid(alpha=0.25)
|
||||
ax.legend(loc="lower right", frameon=True)
|
||||
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=200, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def write_summary(curve_summaries: list[tuple[CurveSpec, dict[str, object]]], out_path: Path) -> None:
|
||||
summary_path = out_path.with_suffix(".json")
|
||||
payload = {
|
||||
"figure": str(out_path),
|
||||
"curves": [
|
||||
{
|
||||
"label": spec.label,
|
||||
"source_kind": spec.source_kind,
|
||||
"mode_dir": spec.mode_dir,
|
||||
"run": spec.run,
|
||||
"tower_path": spec.tower_path,
|
||||
"score_col": spec.score_col,
|
||||
"probs_stem": spec.probs_stem,
|
||||
"auc_mean": float(summary["auc_mean"]),
|
||||
"auc_std": float(summary["auc_std"]),
|
||||
"auc_pooled": float(summary["auc_pooled"]),
|
||||
"fold_count": int(summary["fold_count"]),
|
||||
"n_total": int(summary["n_total"]),
|
||||
}
|
||||
for spec, summary in curve_summaries
|
||||
],
|
||||
}
|
||||
summary_path.write_text(json.dumps(payload, indent=2))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=DEFAULT_OUT,
|
||||
help=f"Output PNG path (default: {DEFAULT_OUT})",
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
||||
curve_summaries: list[tuple[CurveSpec, dict[str, object]]] = []
|
||||
for spec in DEFAULT_CURVES:
|
||||
df, score_col = load_curve_predictions(spec)
|
||||
summary = build_curve_summary(df, score_col)
|
||||
curve_summaries.append((spec, summary))
|
||||
print(
|
||||
f"{spec.label:16s} "
|
||||
f"AUC={summary['auc_mean']:.4f} ± {summary['auc_std']:.4f} "
|
||||
f"pooled={summary['auc_pooled']:.4f} "
|
||||
f"folds={summary['fold_count']}"
|
||||
)
|
||||
|
||||
plot_curves(curve_summaries, args.output)
|
||||
write_summary(curve_summaries, args.output)
|
||||
print(f"Saved figure: {args.output}")
|
||||
print(f"Saved summary: {args.output.with_suffix('.json')}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user