Add new scripts and configurations for model comparison and analysis
- Introduced `poster_model_comparison.py` for generating model comparison figures. - Added `plot_poster_roc_comparison.py` for creating ROC comparison figures for PAPILA binary classification. - Created new JSON configuration files for clinical solo models with and without geometry injection. - Implemented batch dispatch updates in `batch_dispatch.py` to utilize run names from configurations. - Added analysis scripts: `compare_grid.py`, `inspect_embeddings.py`, and `summarize_run.py` for evaluating model performance and feature embeddings. - Created experiment configurations for various training scenarios, including warm sweeps and promoting successful runs. - Added binary ROC comparison and model comparison figures to the results directory.
This commit is contained in:
rpotter6298
@@ -0,0 +1,112 @@
|
||||
"""compare_grid — print a ranked comparison table for a folder of v4 runs.
|
||||
|
||||
A "grid" folder is one whose immediate children are individual run folders, e.g.
|
||||
``v4/results/experiments/tri_v1/grid/`` containing ``bcd35_cw0_nt15/``,
|
||||
``bcd35_cw0_nt25/``, etc. Each child must itself look like a run folder
|
||||
(``repNN/.../summary.json``).
|
||||
|
||||
Usage:
|
||||
python -m v4.scripts.analysis.compare_grid <grid_folder>
|
||||
[--sort {test,val,name,reps}] [--reverse] [--csv]
|
||||
|
||||
Examples:
|
||||
python -m v4.scripts.analysis.compare_grid \
|
||||
v4/results/experiments/tri_v1/grid
|
||||
python -m v4.scripts.analysis.compare_grid \
|
||||
v4/results/experiments/tri_v1 --sort test
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from v4.scripts.analysis.summarize_run import summarise
|
||||
|
||||
|
||||
def collect(grid_dir: Path) -> list[dict]:
|
||||
rows: list[dict] = []
|
||||
for child in sorted(grid_dir.iterdir()):
|
||||
if not child.is_dir():
|
||||
continue
|
||||
s = summarise(child)
|
||||
if s["n_reps"] == 0:
|
||||
continue
|
||||
rows.append({
|
||||
"name": child.name,
|
||||
"n": s["n_reps"],
|
||||
"val_mean": s["val_mean"],
|
||||
"val_std": s["val_std"],
|
||||
"test_mean": s["test_mean"],
|
||||
"test_std": s["test_std"],
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def render_table(rows: list[dict]) -> str:
|
||||
if not rows:
|
||||
return "(no runs found)"
|
||||
name_w = max(len("name"), max(len(r["name"]) for r in rows))
|
||||
header = f"{'name':<{name_w}s} {'reps':>4s} {'val AUC':>17s} {'test AUC':>17s}"
|
||||
sep = "-" * len(header)
|
||||
lines = [header, sep]
|
||||
for r in rows:
|
||||
lines.append(
|
||||
f"{r['name']:<{name_w}s} {r['n']:>4d} "
|
||||
f"{r['val_mean']:.4f} ± {r['val_std']:.4f} "
|
||||
f"{r['test_mean']:.4f} ± {r['test_std']:.4f}"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def render_csv(rows: list[dict]) -> str:
|
||||
buf = sys.stdout
|
||||
w = csv.writer(buf)
|
||||
w.writerow(["name", "reps", "val_mean", "val_std", "test_mean", "test_std"])
|
||||
for r in rows:
|
||||
w.writerow([r["name"], r["n"],
|
||||
f"{r['val_mean']:.6f}", f"{r['val_std']:.6f}",
|
||||
f"{r['test_mean']:.6f}", f"{r['test_std']:.6f}"])
|
||||
return ""
|
||||
|
||||
|
||||
_SORT_KEYS = {
|
||||
"test": lambda r: r["test_mean"],
|
||||
"val": lambda r: r["val_mean"],
|
||||
"name": lambda r: r["name"],
|
||||
"reps": lambda r: r["n"],
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("grid_dir", type=Path,
|
||||
help="Folder whose immediate children are run folders")
|
||||
ap.add_argument("--sort", choices=list(_SORT_KEYS),
|
||||
default="test", help="Sort by which column (default: test)")
|
||||
ap.add_argument("--reverse", action="store_true",
|
||||
help="Reverse the default ordering")
|
||||
ap.add_argument("--csv", action="store_true",
|
||||
help="Emit CSV to stdout instead of a formatted table")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not args.grid_dir.is_dir():
|
||||
raise SystemExit(f"Not a directory: {args.grid_dir}")
|
||||
|
||||
rows = collect(args.grid_dir)
|
||||
descending = args.sort in {"test", "val", "reps"}
|
||||
if args.reverse:
|
||||
descending = not descending
|
||||
rows.sort(key=_SORT_KEYS[args.sort], reverse=descending)
|
||||
|
||||
if args.csv:
|
||||
render_csv(rows)
|
||||
else:
|
||||
print(f"Grid: {args.grid_dir} ({len(rows)} runs, sorted by {args.sort})\n")
|
||||
print(render_table(rows))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,98 @@
|
||||
"""inspect_embeddings — per-dim statistics on a run's saved feature embeddings.
|
||||
|
||||
Loads features.h5 from a run folder (must have been produced with
|
||||
``save_features: true`` in the config), computes per-dimension variance,
|
||||
sparsity (fraction of |z| < threshold), and useful summary stats.
|
||||
|
||||
Helpful for diagnosing whether a fusion stage is collapsing dimensions to
|
||||
near-zero — which would silently null out information when the bridge uses
|
||||
a Hadamard product.
|
||||
|
||||
Usage:
|
||||
python -m v4.scripts.analysis.inspect_embeddings <run_folder> [--phase NAME]
|
||||
[--threshold 0.05]
|
||||
|
||||
Examples:
|
||||
python -m v4.scripts.analysis.inspect_embeddings \
|
||||
v4/results/experiments/tri_v1/baseline_tri/rep00 --phase nt
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import h5py
|
||||
import numpy as np
|
||||
|
||||
|
||||
def load_phase(features_path: Path, phase: str | None):
|
||||
with h5py.File(features_path, "r") as f:
|
||||
phases = list(f.keys())
|
||||
if phase is None:
|
||||
phase = phases[-1]
|
||||
if phase not in phases:
|
||||
raise SystemExit(
|
||||
f"Phase {phase!r} not in {features_path} (available: {phases})"
|
||||
)
|
||||
g = f[phase]
|
||||
z = g["z"][:] # (n_folds, n_samples, n_dim)
|
||||
split = g["split"][:].astype(str) # (n_folds, n_samples)
|
||||
y_true = g["y_true"][:]
|
||||
return phase, z, split, y_true, phases
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("run_dir", type=Path, help="Folder containing features.h5 (under repNN)")
|
||||
ap.add_argument("--phase", default=None,
|
||||
help="Bridge phase name (e.g. nt, hb, cd_fuse). Default: last in file.")
|
||||
ap.add_argument("--threshold", type=float, default=0.05,
|
||||
help="|z| threshold for the sparsity count (default: 0.05)")
|
||||
args = ap.parse_args()
|
||||
|
||||
feat = next(iter(args.run_dir.rglob("features.h5")), None)
|
||||
if feat is None:
|
||||
raise SystemExit(f"No features.h5 found under {args.run_dir}")
|
||||
|
||||
phase, z, split, _, all_phases = load_phase(feat, args.phase)
|
||||
|
||||
val_mask = (split == "val")
|
||||
z_val = z[val_mask] # (n_val_total, n_dim)
|
||||
z_train = z[(split == "train")]
|
||||
n_dim = z_val.shape[-1]
|
||||
|
||||
print(f"Run: {args.run_dir}")
|
||||
print(f"File: {feat}")
|
||||
print(f"Phases: {all_phases}")
|
||||
print(f"Phase: {phase} (z shape: {z.shape})")
|
||||
print(f"Split sizes: train={len(z_train)} val={len(z_val)}")
|
||||
print()
|
||||
|
||||
abs_z = np.abs(z_val)
|
||||
per_dim_var = z_val.var(axis=0)
|
||||
per_dim_abs = abs_z.mean(axis=0)
|
||||
per_dim_max = abs_z.max(axis=0)
|
||||
sparsity = (abs_z < args.threshold).mean(axis=0)
|
||||
|
||||
print(f"Aggregate stats over val embeddings (|z| < {args.threshold} = 'near-zero'):")
|
||||
print(f" global mean(|z|): {abs_z.mean():.4f}")
|
||||
print(f" global var(z): {z_val.var():.4f}")
|
||||
print(f" fraction near-zero (global): {(abs_z < args.threshold).mean():.4f}")
|
||||
print()
|
||||
print(f"Per-dimension summary ({n_dim} dims):")
|
||||
print(f" variance: min={per_dim_var.min():.4f} med={np.median(per_dim_var):.4f} max={per_dim_var.max():.4f}")
|
||||
print(f" mean |z|: min={per_dim_abs.min():.4f} med={np.median(per_dim_abs):.4f} max={per_dim_abs.max():.4f}")
|
||||
print(f" near-zero rate: min={sparsity.min():.4f} med={np.median(sparsity):.4f} max={sparsity.max():.4f}")
|
||||
|
||||
# Dead dimensions: high near-zero rate
|
||||
dead = np.where(sparsity > 0.9)[0]
|
||||
print(f" 'dead' dims (>90% near-zero): {len(dead)}/{n_dim} "
|
||||
f"{('idx: ' + str(dead.tolist())) if 0 < len(dead) <= 20 else ''}")
|
||||
weak = np.where(per_dim_var < 1e-4)[0]
|
||||
print(f" 'weak' dims (var < 1e-4): {len(weak)}/{n_dim} "
|
||||
f"{('idx: ' + str(weak.tolist())) if 0 < len(weak) <= 20 else ''}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,132 @@
|
||||
"""summarize_run — print cross-rep stats for one v4 run folder.
|
||||
|
||||
A "run" is a folder like ``v4/results/experiments/tri_v1/grid/bcd75_cw1_nt25/``
|
||||
containing ``rep00/``, ``rep01/``, ... — each with a per-rep ``summary.json``
|
||||
under any ``out_dir_tags`` subdir (typically ``binary/`` or ``binary/ntower/``).
|
||||
|
||||
Usage:
|
||||
python -m v4.scripts.analysis.summarize_run <run_folder> [--per-rep] [--json]
|
||||
|
||||
Examples:
|
||||
python -m v4.scripts.analysis.summarize_run \
|
||||
v4/results/experiments/tri_v1/grid/bcd75_cw1_nt25
|
||||
python -m v4.scripts.analysis.summarize_run \
|
||||
v4/results/experiments/tri_v1/baseline_tri --per-rep
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def find_rep_summaries(run_dir: Path) -> list[tuple[int, Path]]:
|
||||
"""Return [(rep_idx, summary_path), ...] sorted by rep_idx."""
|
||||
out: list[tuple[int, Path]] = []
|
||||
for rep_dir in sorted(run_dir.glob("rep*")):
|
||||
if not rep_dir.is_dir():
|
||||
continue
|
||||
m = re.match(r"rep(\d+)$", rep_dir.name)
|
||||
if not m:
|
||||
continue
|
||||
summary = next(iter(rep_dir.rglob("summary.json")), None)
|
||||
if summary is not None:
|
||||
out.append((int(m.group(1)), summary))
|
||||
return out
|
||||
|
||||
|
||||
def load_rep(summary_path: Path) -> dict:
|
||||
"""Extract the fields we summarise from one rep's summary.json."""
|
||||
d = json.loads(summary_path.read_text())
|
||||
return {
|
||||
"val_mean": float(d.get("mean_val_auc", float("nan"))),
|
||||
"val_std": float(d.get("std_val_auc", float("nan"))),
|
||||
"test_mean": float(d.get("mean_test_auc", float("nan"))),
|
||||
"test_std": float(d.get("std_test_auc", float("nan"))),
|
||||
"elapsed_s": float(d.get("elapsed_s", float("nan"))),
|
||||
"eval_stage": d.get("eval_stage", "?"),
|
||||
}
|
||||
|
||||
|
||||
def summarise(run_dir: Path) -> dict:
|
||||
reps = find_rep_summaries(run_dir)
|
||||
if not reps:
|
||||
return {"run": str(run_dir), "n_reps": 0, "reps": []}
|
||||
rows = [(idx, load_rep(p)) for idx, p in reps]
|
||||
val = np.array([r[1]["val_mean"] for r in rows])
|
||||
test = np.array([r[1]["test_mean"] for r in rows])
|
||||
elaps = np.array([r[1]["elapsed_s"] for r in rows])
|
||||
out = {
|
||||
"run": str(run_dir),
|
||||
"n_reps": len(rows),
|
||||
"eval_stage": rows[0][1]["eval_stage"],
|
||||
"val_mean": float(np.mean(val)),
|
||||
"val_std": float(np.std(val)),
|
||||
"val_min": float(np.min(val)),
|
||||
"val_max": float(np.max(val)),
|
||||
"test_mean": float(np.mean(test)),
|
||||
"test_std": float(np.std(test)),
|
||||
"test_min": float(np.min(test)),
|
||||
"test_max": float(np.max(test)),
|
||||
"elapsed_total_s": float(np.sum(elaps)) if not np.isnan(elaps).any() else None,
|
||||
"reps": [
|
||||
{"rep": idx, **info} for idx, info in rows
|
||||
],
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def render(s: dict, per_rep: bool = False) -> str:
|
||||
if s["n_reps"] == 0:
|
||||
return f"Run: {s['run']}\n no reps with summary.json found."
|
||||
|
||||
lines = [
|
||||
f"Run: {s['run']}",
|
||||
f"Reps: {s['n_reps']} (eval_stage={s['eval_stage']})",
|
||||
f"Val AUC: {s['val_mean']:.4f} ± {s['val_std']:.4f} "
|
||||
f"[min={s['val_min']:.4f} max={s['val_max']:.4f}]",
|
||||
f"Test AUC: {s['test_mean']:.4f} ± {s['test_std']:.4f} "
|
||||
f"[min={s['test_min']:.4f} max={s['test_max']:.4f}]",
|
||||
]
|
||||
if s.get("elapsed_total_s") is not None:
|
||||
h = s["elapsed_total_s"] / 3600
|
||||
lines.append(f"Compute: {s['elapsed_total_s']:.0f} s total ({h:.1f} h)")
|
||||
|
||||
if per_rep:
|
||||
lines.append("")
|
||||
lines.append("Per-rep breakdown:")
|
||||
lines.append(f" {'rep':>4s} {'val':>7s} {'test':>7s} {'elapsed':>7s}")
|
||||
for r in s["reps"]:
|
||||
elapsed = (f"{r['elapsed_s']:.0f}s" if not np.isnan(r['elapsed_s']) else "-")
|
||||
lines.append(
|
||||
f" {r['rep']:>4d} "
|
||||
f"{r['val_mean']:>7.4f} "
|
||||
f"{r['test_mean']:>7.4f} "
|
||||
f"{elapsed:>7s}"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("run_dir", type=Path, help="Path to a run folder (contains repNN/ subdirs)")
|
||||
ap.add_argument("--per-rep", action="store_true", help="Print one line per rep")
|
||||
ap.add_argument("--json", action="store_true", help="Emit JSON instead of formatted text")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not args.run_dir.is_dir():
|
||||
raise SystemExit(f"Not a directory: {args.run_dir}")
|
||||
|
||||
s = summarise(args.run_dir)
|
||||
if args.json:
|
||||
print(json.dumps(s, indent=2))
|
||||
else:
|
||||
print(render(s, per_rep=args.per_rep))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,10 @@
|
||||
[
|
||||
{
|
||||
"_note": "Single rep of tritower default with save_features=true so we can inspect nt-stage embeddings (Hadamard-product collapse hypothesis).",
|
||||
"run_name": "experiments/tri_v1/baseline_tri_features",
|
||||
"reps": 1,
|
||||
"overrides": {
|
||||
"save_features": true
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,13 @@
|
||||
[
|
||||
{ "_note": "cd-solo + UNet geom-vector inject; cd_warm sweep. 3 reps each.",
|
||||
"run_name": "experiments/tri_v1/cd_solo_geom/warm00", "reps": 3,
|
||||
"stage_overrides": { "cd_warm": { "epochs": 0 } } },
|
||||
{ "run_name": "experiments/tri_v1/cd_solo_geom/warm05", "reps": 3,
|
||||
"stage_overrides": { "cd_warm": { "epochs": 5 } } },
|
||||
{ "run_name": "experiments/tri_v1/cd_solo_geom/warm10", "reps": 3,
|
||||
"stage_overrides": { "cd_warm": { "epochs": 10 } } },
|
||||
{ "run_name": "experiments/tri_v1/cd_solo_geom/warm20", "reps": 3,
|
||||
"stage_overrides": { "cd_warm": { "epochs": 20 } } },
|
||||
{ "run_name": "experiments/tri_v1/cd_solo_geom/warm40", "reps": 3,
|
||||
"stage_overrides": { "cd_warm": { "epochs": 40 } } }
|
||||
]
|
||||
@@ -0,0 +1,13 @@
|
||||
[
|
||||
{ "_note": "cd-solo, no geom; cd_warm sweep. 3 reps each. Companion to cd_solo_geom sweep.",
|
||||
"run_name": "experiments/tri_v1/cd_solo/warm00", "reps": 3,
|
||||
"stage_overrides": { "cd_warm": { "epochs": 0 } } },
|
||||
{ "run_name": "experiments/tri_v1/cd_solo/warm05", "reps": 3,
|
||||
"stage_overrides": { "cd_warm": { "epochs": 5 } } },
|
||||
{ "run_name": "experiments/tri_v1/cd_solo/warm10", "reps": 3,
|
||||
"stage_overrides": { "cd_warm": { "epochs": 10 } } },
|
||||
{ "run_name": "experiments/tri_v1/cd_solo/warm20", "reps": 3,
|
||||
"stage_overrides": { "cd_warm": { "epochs": 20 } } },
|
||||
{ "run_name": "experiments/tri_v1/cd_solo/warm40", "reps": 3,
|
||||
"stage_overrides": { "cd_warm": { "epochs": 40 } } }
|
||||
]
|
||||
@@ -0,0 +1,16 @@
|
||||
[
|
||||
{
|
||||
"_note": "Promote grid winner bcd75_cw1_nt25 (3-rep test AUC 0.910) to 10 reps. First 3 reps will be skipped (already on disk).",
|
||||
"run_name": "experiments/tri_v1/grid/bcd75_cw1_nt25",
|
||||
"reps": 10,
|
||||
"overrides": {
|
||||
"training": {
|
||||
"bcd_prob": 0.75,
|
||||
"class_weighted": true
|
||||
}
|
||||
},
|
||||
"stage_overrides": {
|
||||
"nt": { "epochs": 25 }
|
||||
}
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user