Add new scripts and configurations for model comparison and analysis

- Introduced `poster_model_comparison.py` for generating model comparison figures.
- Added `plot_poster_roc_comparison.py` for creating ROC comparison figures for PAPILA binary classification.
- Created new JSON configuration files for clinical solo models with and without geometry injection.
- Implemented batch dispatch updates in `batch_dispatch.py` to utilize run names from configurations.
- Added analysis scripts: `compare_grid.py`, `inspect_embeddings.py`, and `summarize_run.py` for evaluating model performance and feature embeddings.
- Created experiment configurations for various training scenarios, including warm sweeps and promoting successful runs.
- Added binary ROC comparison and model comparison figures to the results directory.
This commit is contained in:
rpotter6298
2026-05-14 13:43:30 +02:00
parent e2489a21e7
commit a721e52909
18 changed files with 1052 additions and 1 deletions
+112
View File
@@ -0,0 +1,112 @@
"""compare_grid — print a ranked comparison table for a folder of v4 runs.
A "grid" folder is one whose immediate children are individual run folders, e.g.
``v4/results/experiments/tri_v1/grid/`` containing ``bcd35_cw0_nt15/``,
``bcd35_cw0_nt25/``, etc. Each child must itself look like a run folder
(``repNN/.../summary.json``).
Usage:
python -m v4.scripts.analysis.compare_grid <grid_folder>
[--sort {test,val,name,reps}] [--reverse] [--csv]
Examples:
python -m v4.scripts.analysis.compare_grid \
v4/results/experiments/tri_v1/grid
python -m v4.scripts.analysis.compare_grid \
v4/results/experiments/tri_v1 --sort test
"""
from __future__ import annotations
import argparse
import csv
import sys
from pathlib import Path
from v4.scripts.analysis.summarize_run import summarise
def collect(grid_dir: Path) -> list[dict]:
rows: list[dict] = []
for child in sorted(grid_dir.iterdir()):
if not child.is_dir():
continue
s = summarise(child)
if s["n_reps"] == 0:
continue
rows.append({
"name": child.name,
"n": s["n_reps"],
"val_mean": s["val_mean"],
"val_std": s["val_std"],
"test_mean": s["test_mean"],
"test_std": s["test_std"],
})
return rows
def render_table(rows: list[dict]) -> str:
if not rows:
return "(no runs found)"
name_w = max(len("name"), max(len(r["name"]) for r in rows))
header = f"{'name':<{name_w}s} {'reps':>4s} {'val AUC':>17s} {'test AUC':>17s}"
sep = "-" * len(header)
lines = [header, sep]
for r in rows:
lines.append(
f"{r['name']:<{name_w}s} {r['n']:>4d} "
f"{r['val_mean']:.4f} ± {r['val_std']:.4f} "
f"{r['test_mean']:.4f} ± {r['test_std']:.4f}"
)
return "\n".join(lines)
def render_csv(rows: list[dict]) -> str:
buf = sys.stdout
w = csv.writer(buf)
w.writerow(["name", "reps", "val_mean", "val_std", "test_mean", "test_std"])
for r in rows:
w.writerow([r["name"], r["n"],
f"{r['val_mean']:.6f}", f"{r['val_std']:.6f}",
f"{r['test_mean']:.6f}", f"{r['test_std']:.6f}"])
return ""
_SORT_KEYS = {
"test": lambda r: r["test_mean"],
"val": lambda r: r["val_mean"],
"name": lambda r: r["name"],
"reps": lambda r: r["n"],
}
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("grid_dir", type=Path,
help="Folder whose immediate children are run folders")
ap.add_argument("--sort", choices=list(_SORT_KEYS),
default="test", help="Sort by which column (default: test)")
ap.add_argument("--reverse", action="store_true",
help="Reverse the default ordering")
ap.add_argument("--csv", action="store_true",
help="Emit CSV to stdout instead of a formatted table")
args = ap.parse_args()
if not args.grid_dir.is_dir():
raise SystemExit(f"Not a directory: {args.grid_dir}")
rows = collect(args.grid_dir)
descending = args.sort in {"test", "val", "reps"}
if args.reverse:
descending = not descending
rows.sort(key=_SORT_KEYS[args.sort], reverse=descending)
if args.csv:
render_csv(rows)
else:
print(f"Grid: {args.grid_dir} ({len(rows)} runs, sorted by {args.sort})\n")
print(render_table(rows))
if __name__ == "__main__":
main()