ryan/hypertower
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add new scripts and configurations for model comparison and analysis

Browse Source 
- Introduced `poster_model_comparison.py` for generating model comparison figures.
- Added `plot_poster_roc_comparison.py` for creating ROC comparison figures for PAPILA binary classification.
- Created new JSON configuration files for clinical solo models with and without geometry injection.
- Implemented batch dispatch updates in `batch_dispatch.py` to utilize run names from configurations.
- Added analysis scripts: `compare_grid.py`, `inspect_embeddings.py`, and `summarize_run.py` for evaluating model performance and feature embeddings.
- Created experiment configurations for various training scenarios, including warm sweeps and promoting successful runs.
- Added binary ROC comparison and model comparison figures to the results directory.
This commit is contained in:
rpotter6298
2026-05-14 13:43:30 +02:00
parent e2489a21e7
commit a721e52909
18 changed files with 1052 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
results/poster/papila_binary_roc_comparison.json
+61
View File
@@ -0,0 +1,61 @@
{
"figure": "/home/rpotter/hypertower/results/poster/papila_binary_roc_comparison.png",
"curves": [
{
"label": "Clinical Data Only",
"source_kind": "legacy_npy",
"mode_dir": "analysis_data/pipeline_mdonly_500ep/binary/single",
"run": null,
"tower_path": null,
"score_col": null,
"probs_stem": "probs_classic",
"auc_mean": 0.7311688311688311,
"auc_std": 0.08461127279353979,
"auc_pooled": 0.6812987012987013,
"fold_count": 5,
"n_total": 400
},
{
"label": "Image Only",
"source_kind": "v3_csv",
"mode_dir": null,
"run": "phase2/imageonly_resnet50_proper",
"tower_path": "binary/single",
"score_col": "prob_img_c1",
"probs_stem": null,
"auc_mean": 0.819264705882353,
"auc_std": 0.0740851608018641,
"auc_pooled": 0.8099584558823529,
"fold_count": 50,
"n_total": 4200
},
{
"label": "Single Fusion",
"source_kind": "v3_csv",
"mode_dir": null,
"run": "phase5/single_fused",
"tower_path": "binary/single",
"score_col": "prob_fused_c1",
"probs_stem": null,
"auc_mean": 0.8506801470588234,
"auc_std": 0.07392522372615995,
"auc_pooled": 0.8402091911764706,
"fold_count": 50,
"n_total": 4200
},
{
"label": "Ensemble Fusion",
"source_kind": "v3_csv",
"mode_dir": null,
"run": "phase5/logit_mlp_head",
"tower_path": "binary/ensemble",
"score_col": "prob_fused_c1",
"probs_stem": null,
"auc_mean": 0.8988235294117648,
"auc_std": 0.06435919826554772,
"auc_pooled": 0.8855705882352942,
"fold_count": 50,
"n_total": 2100
}
]
}
results/poster/papila_binary_roc_comparison.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 219 KiB

v3/figures/explainability/poster_model_comparison.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 187 KiB

v3/scripts/output_analysis/explainability/poster_model_comparison.py
+132
View File
@@ -0,0 +1,132 @@
#!/usr/bin/env python3
"""
Poster-facing model comparison figure in the style of head_comparison.png.
Compares:
- MD Only
- Image Only
- Ensemble Fusion
using the same run sources as the poster ROC comparison script.
"""
from __future__ import annotations
import argparse
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
from v3.scripts.output_analysis.plot_poster_roc_comparison import (
DEFAULT_CURVES,
REPO_ROOT,
build_curve_summary,
load_curve_predictions,
)
DEFAULT_OUT = REPO_ROOT / "v3" / "figures" / "explainability" / "poster_model_comparison.png"
MODEL_ORDER = ["Clinical Data Only", "Image Only", "Ensemble Fusion"]
C_NORMAL = "#4c72b0"
C_GLAUCOMA = "#c44e52"
BG = "#e8e8e8"
def _select_specs():
selected = []
for label in MODEL_ORDER:
matches = [spec for spec in DEFAULT_CURVES if spec.label == label]
if not matches:
raise ValueError(f"Could not find default poster curve for '{label}'")
selected.append(matches[0])
return selected
def make_model_comparison(out_path: Path) -> None:
specs = _select_specs()
rng = np.random.default_rng(42)
fig, axes = plt.subplots(1, len(specs), figsize=(11, 4.5), sharey=True)
if len(specs) == 1:
axes = [axes]
fig.patch.set_facecolor(BG)
fig.suptitle(
"Model Comparison — P(Glaucoma) by True Class",
fontsize=12,
fontweight="bold",
)
for ax, spec in zip(axes, specs):
df, score_col = load_curve_predictions(spec)
summary = build_curve_summary(df, score_col)
ax.set_facecolor(BG)
data_by_class = [df.loc[df["y_true"] == cls, score_col].values for cls in [0, 1]]
vp = ax.violinplot(
data_by_class,
positions=[0, 1],
widths=0.6,
showmedians=True,
showextrema=False,
)
for body, color in zip(vp["bodies"], [C_NORMAL, C_GLAUCOMA]):
body.set_facecolor(color)
body.set_alpha(0.35)
vp["cmedians"].set_color("#222")
vp["cmedians"].set_linewidth(2)
for cls, color in zip([0, 1], [C_NORMAL, C_GLAUCOMA]):
vals = data_by_class[cls]
jitter = rng.uniform(-0.12, 0.12, len(vals))
ax.scatter(
cls + jitter,
vals,
color=color,
s=4,
alpha=0.30,
linewidths=0,
zorder=3,
)
ax.axhline(0.5, color="#888", lw=1.0, ls="--", alpha=0.6)
ax.set_xticks([0, 1])
ax.set_xticklabels(["Normal", "Glaucoma"], fontsize=9)
ax.set_title(spec.label, fontsize=10, fontweight="bold", color=spec.color)
ax.set_ylim(-0.05, 1.05)
ax.grid(axis="y", alpha=0.3)
if ax is axes[0]:
ax.set_ylabel("Predicted P(Glaucoma)", fontsize=10)
ax.text(
0.97,
0.04,
f"AUC = {summary['auc_mean']:.3f} ± {summary['auc_std']:.3f}",
transform=ax.transAxes,
ha="right",
va="bottom",
fontsize=9,
color="#333",
bbox=dict(facecolor="white", alpha=0.65, edgecolor="none", pad=2),
)
fig.tight_layout()
out_path.parent.mkdir(parents=True, exist_ok=True)
fig.savefig(out_path, dpi=180, bbox_inches="tight")
plt.close(fig)
print(f"Saved: {out_path}")
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--out", type=Path, default=DEFAULT_OUT)
args = ap.parse_args()
make_model_comparison(args.out)
if __name__ == "__main__":
main()
v3/scripts/output_analysis/plot_poster_roc_comparison.py
+283
View File
@@ -0,0 +1,283 @@
#!/usr/bin/env python3
"""
Poster-facing ROC comparison for PAPILA binary glaucoma classification.
Builds one combined ROC figure with four curves:
- MD Only
- Image Only
- Single Fusion
- Ensemble Fusion
The default setup mixes two artifact layouts:
1. Dedicated unimodal runs saved under analysis_data/.../fold*/{y_true,probs}.npy
2. Repeated CV fusion runs saved under v3/results/.../rep*/.../predictions_test.csv
Update DEFAULT_CURVES below if you want different source runs or labels.
"""
from __future__ import annotations
import argparse
import json
from dataclasses import dataclass
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.metrics import roc_auc_score, roc_curve
REPO_ROOT = Path(__file__).resolve().parents[3]
RESULTS_ROOT = REPO_ROOT / "v3" / "results"
DEFAULT_OUT = REPO_ROOT / "results" / "poster" / "papila_binary_roc_comparison.png"
@dataclass(frozen=True)
class CurveSpec:
label: str
source_kind: str
color: str
mode_dir: str | None = None
probs_stem: str | None = None
run: str | None = None
tower_path: str | None = None
score_col: str | None = None
DEFAULT_CURVES = [
CurveSpec(
label="Clinical Data Only",
source_kind="legacy_npy",
color="#4c72b0",
mode_dir="analysis_data/pipeline_mdonly_500ep/binary/single",
probs_stem="probs_classic",
),
CurveSpec(
label="Image Only",
source_kind="v3_csv",
color="#dd8452",
run="phase2/imageonly_resnet50_proper",
tower_path="binary/single",
score_col="prob_img_c1",
),
CurveSpec(
label="Single Fusion",
source_kind="v3_csv",
color="#55a868",
run="phase5/single_fused",
tower_path="binary/single",
score_col="prob_fused_c1",
),
CurveSpec(
label="Ensemble Fusion",
source_kind="v3_csv",
color="#c44e52",
run="phase5/logit_mlp_head",
tower_path="binary/ensemble",
score_col="prob_fused_c1",
),
]
def load_v3_csv_predictions(run: str, tower_path: str, score_col: str) -> pd.DataFrame:
run_dir = RESULTS_ROOT / run
rows: list[pd.DataFrame] = []
for rep_dir in sorted(run_dir.glob("rep*")):
mode_dir = rep_dir / tower_path
if not mode_dir.exists():
continue
for fold_dir in sorted(mode_dir.glob("fold[0-9]")):
csv_path = fold_dir / "predictions_test.csv"
if not csv_path.exists():
continue
df = pd.read_csv(csv_path, usecols=["y_true", score_col])
df["rep"] = rep_dir.name
df["fold"] = fold_dir.name
rows.append(df)
if not rows:
raise FileNotFoundError(f"No predictions found under {run_dir}/{tower_path}")
return pd.concat(rows, ignore_index=True)
def load_legacy_npy_predictions(mode_dir: str, probs_stem: str) -> pd.DataFrame:
root = REPO_ROOT / mode_dir
rows: list[pd.DataFrame] = []
for fold_dir in sorted(root.glob("fold[0-9]")):
y_path = fold_dir / "y_true.npy"
p_path = fold_dir / f"{probs_stem}.npy"
if not y_path.exists() or not p_path.exists():
continue
y_true = np.load(y_path)
probs = np.load(p_path)
if probs.ndim == 2:
if probs.shape[1] < 2:
raise ValueError(f"Expected 2-class probs in {p_path}")
scores = probs[:, 1]
else:
scores = probs
rows.append(
pd.DataFrame(
{
"y_true": y_true,
"score": scores,
"rep": "rep00",
"fold": fold_dir.name,
}
)
)
if not rows:
raise FileNotFoundError(f"No fold artifacts found under {root}")
return pd.concat(rows, ignore_index=True)
def load_curve_predictions(spec: CurveSpec) -> tuple[pd.DataFrame, str]:
if spec.source_kind == "v3_csv":
if spec.run is None or spec.tower_path is None or spec.score_col is None:
raise ValueError(f"Incomplete v3_csv spec: {spec}")
return load_v3_csv_predictions(spec.run, spec.tower_path, spec.score_col), spec.score_col
if spec.source_kind == "legacy_npy":
if spec.mode_dir is None or spec.probs_stem is None:
raise ValueError(f"Incomplete legacy_npy spec: {spec}")
return load_legacy_npy_predictions(spec.mode_dir, spec.probs_stem), "score"
raise ValueError(f"Unknown source_kind: {spec.source_kind}")
def build_curve_summary(df: pd.DataFrame, score_col: str) -> dict[str, object]:
if score_col not in df.columns:
raise KeyError(f"Missing score column '{score_col}'")
mean_fpr = np.linspace(0.0, 1.0, 501)
tprs: list[np.ndarray] = []
aucs: list[float] = []
fold_count = 0
for (_, _), fold_df in df.groupby(["rep", "fold"], sort=True):
y_true = fold_df["y_true"].to_numpy()
scores = fold_df[score_col].to_numpy()
if len(np.unique(y_true)) < 2:
continue
fpr, tpr, _ = roc_curve(y_true, scores)
interp_tpr = np.interp(mean_fpr, fpr, tpr)
interp_tpr[0] = 0.0
interp_tpr[-1] = 1.0
tprs.append(interp_tpr)
aucs.append(float(roc_auc_score(y_true, scores)))
fold_count += 1
if not tprs:
raise ValueError(f"No valid binary folds found for '{score_col}'")
tpr_arr = np.vstack(tprs)
y_all = df["y_true"].to_numpy()
s_all = df[score_col].to_numpy()
return {
"fpr": mean_fpr,
"tpr_mean": tpr_arr.mean(axis=0),
"tpr_std": tpr_arr.std(axis=0),
"auc_mean": float(np.mean(aucs)),
"auc_std": float(np.std(aucs)),
"auc_pooled": float(roc_auc_score(y_all, s_all)),
"fold_count": fold_count,
"n_total": int(len(df)),
}
def plot_curves(curve_summaries: list[tuple[CurveSpec, dict[str, object]]], out_path: Path) -> None:
out_path.parent.mkdir(parents=True, exist_ok=True)
fig, ax = plt.subplots(figsize=(8.8, 7.0))
ax.plot([0, 1], [0, 1], linestyle="--", linewidth=1, color="0.5", alpha=0.8)
for spec, summary in curve_summaries:
fpr = np.asarray(summary["fpr"])
tpr_mean = np.asarray(summary["tpr_mean"])
tpr_std = np.asarray(summary["tpr_std"])
auc_mean = float(summary["auc_mean"])
auc_std = float(summary["auc_std"])
ax.plot(
fpr,
tpr_mean,
color=spec.color,
linewidth=2.4,
label=f"{spec.label} (AUC {auc_mean:.3f} ± {auc_std:.3f})",
)
ax.fill_between(
fpr,
np.clip(tpr_mean - tpr_std, 0, 1),
np.clip(tpr_mean + tpr_std, 0, 1),
color=spec.color,
alpha=0.12,
)
ax.set_xlim(-0.01, 1.01)
ax.set_ylim(-0.01, 1.01)
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title("PAPILA Binary ROC Comparison")
ax.grid(alpha=0.25)
ax.legend(loc="lower right", frameon=True)
fig.tight_layout()
fig.savefig(out_path, dpi=200, bbox_inches="tight")
plt.close(fig)
def write_summary(curve_summaries: list[tuple[CurveSpec, dict[str, object]]], out_path: Path) -> None:
summary_path = out_path.with_suffix(".json")
payload = {
"figure": str(out_path),
"curves": [
{
"label": spec.label,
"source_kind": spec.source_kind,
"mode_dir": spec.mode_dir,
"run": spec.run,
"tower_path": spec.tower_path,
"score_col": spec.score_col,
"probs_stem": spec.probs_stem,
"auc_mean": float(summary["auc_mean"]),
"auc_std": float(summary["auc_std"]),
"auc_pooled": float(summary["auc_pooled"]),
"fold_count": int(summary["fold_count"]),
"n_total": int(summary["n_total"]),
}
for spec, summary in curve_summaries
],
}
summary_path.write_text(json.dumps(payload, indent=2))
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument(
"--output",
type=Path,
default=DEFAULT_OUT,
help=f"Output PNG path (default: {DEFAULT_OUT})",
)
args = ap.parse_args()
curve_summaries: list[tuple[CurveSpec, dict[str, object]]] = []
for spec in DEFAULT_CURVES:
df, score_col = load_curve_predictions(spec)
summary = build_curve_summary(df, score_col)
curve_summaries.append((spec, summary))
print(
f"{spec.label:16s} "
f"AUC={summary['auc_mean']:.4f} ± {summary['auc_std']:.4f} "
f"pooled={summary['auc_pooled']:.4f} "
f"folds={summary['fold_count']}"
)
plot_curves(curve_summaries, args.output)
write_summary(curve_summaries, args.output)
print(f"Saved figure: {args.output}")
print(f"Saved summary: {args.output.with_suffix('.json')}")
if __name__ == "__main__":
main()
v4/configs/clinical_solo.json
+78
View File
@@ -0,0 +1,78 @@
{
"_notes": [
"Diagnostic: cd tower alone, no geometry injection.",
"Pair to clinical_solo_geom_unet — same architecture but with geom_dim=0",
"and no img EPC supplier."
],
"run_name": "v4/clinical_solo",
"num_classes": 2,
"label_filter": [0, 1],
"split_identity_level": 1,
"eval_stage": "cd_fuse",
"save_predictions": true,
"save_features": true,
"seed": 1234,
"folds": 5,
"fold_seed": 100,
"output_root": "v4/results",
"out_dir_tags": ["binary"],
"data": {
"module": "v4.classes.profiles.v4papila",
"args": {
"image_dir": "Papila/FundusImages",
"clinical_dir": "Papila/ClinicalData",
"label_col": "Diagnosis",
"iop_corr_method": "ratio",
"iop_drop_raw": true,
"exclude_cols": ["Axial_Length"],
"in_memory_cache": false
}
},
"towers": [
{
"name": "cd",
"module": "v4.classes.towers.clinical_tower",
"class": "ClinicalEncoder",
"data_source": "matrix",
"args": {
"hidden_dim": 128
}
}
],
"stages": [
{
"name": "cd_warm",
"type": "warm",
"tower": "cd",
"head_name": "cd_aux",
"level": "eye",
"epochs": 40
},
{
"name": "cd_aux",
"type": "head",
"input": "cd",
"train_with": "cd_fuse"
},
{
"name": "cd_fuse",
"type": "fusion",
"module": "v4.classes.bridges.mono_bridge",
"class": "MonoBridge",
"inputs": ["cd"],
"level": "eye",
"epochs": 36,
"train_towers": true,
"args": { "use_ln": false }
}
],
"training": {
"lr": 1e-4,
"batch_size": 16,
"tune_binary_threshold": true
}
}
v4/configs/clinical_solo_geom_unet.json
+103
View File
@@ -0,0 +1,103 @@
{
"_notes": [
"Diagnostic: cd tower alone, with UNet-derived 5-feature CDR vector injection.",
"img tower is present only as an EPC supplier — its early_pass runs the UNet",
"fine-tune+inference pipeline and publishes geometry_vectors, but no stage",
"uses its embedding so its CNN body sits idle.",
"cd consumes the vectors via epc_requests and trains under MonoBridge."
],
"run_name": "v4/clinical_solo_geom_unet",
"num_classes": 2,
"label_filter": [0, 1],
"split_identity_level": 1,
"eval_stage": "cd_fuse",
"save_predictions": true,
"save_features": true,
"seed": 1234,
"folds": 5,
"fold_seed": 100,
"output_root": "v4/results",
"out_dir_tags": ["binary"],
"data": {
"module": "v4.classes.profiles.v4papila",
"args": {
"image_dir": "Papila/FundusImages",
"clinical_dir": "Papila/ClinicalData",
"label_col": "Diagnosis",
"iop_corr_method": "ratio",
"iop_drop_raw": true,
"exclude_cols": ["Axial_Length"],
"in_memory_cache": false
}
},
"towers": [
{
"name": "img",
"module": "v4.classes.towers.image_tower",
"class": "ImageEncoder",
"data_source": "image",
"epc_supplies": ["geometry_vectors"],
"args": {
"backbone": "refugelike",
"freeze_ratio": 1.0,
"augment": false,
"geometry_source": "unet",
"weights_path": "models/v2/refuge/segmentation/per_image/best.pt",
"contour_dir": "Papila/ExpertsSegmentations/Contours",
"unet_size": 512,
"normalize": "per_image",
"threshold": 0.5,
"finetune_epochs": 10,
"finetune_lr": 1e-5,
"finetune_batch_size": 4
}
},
{
"name": "cd",
"module": "v4.classes.towers.clinical_tower",
"class": "ClinicalEncoder",
"data_source": "matrix",
"epc_requests": ["geometry_vectors"],
"args": {
"hidden_dim": 128,
"geom_dim": 5
}
}
],
"stages": [
{
"name": "cd_warm",
"type": "warm",
"tower": "cd",
"head_name": "cd_aux",
"level": "eye",
"epochs": 40
},
{
"name": "cd_aux",
"type": "head",
"input": "cd",
"train_with": "cd_fuse"
},
{
"name": "cd_fuse",
"type": "fusion",
"module": "v4.classes.bridges.mono_bridge",
"class": "MonoBridge",
"inputs": ["cd"],
"level": "eye",
"epochs": 36,
"train_towers": true,
"args": { "use_ln": false }
}
],
"training": {
"lr": 1e-4,
"batch_size": 16,
"tune_binary_threshold": true
}
}
v4/distributed/batch_dispatch.py
+1 -1
View File
@@ -169,7 +169,7 @@ def dispatch_batch(
server_cfg_path = str(rel_path)
job_body = {
"run_name": run_name,
"run_name": cfg["run_name"], # per-rep, e.g. ".../rep05"
"module": "v4.classes.v4_hypertower",
"args": ["--config", server_cfg_path],
"output_dir": output_root,
v4/distributed/jobs.db
BIN
View File
Binary file not shown.
v4/scripts/__init__.py
View File
v4/scripts/analysis/__init__.py
View File
v4/scripts/analysis/compare_grid.py
+112
View File
@@ -0,0 +1,112 @@
"""compare_grid — print a ranked comparison table for a folder of v4 runs.
A "grid" folder is one whose immediate children are individual run folders, e.g.
``v4/results/experiments/tri_v1/grid/`` containing ``bcd35_cw0_nt15/``,
``bcd35_cw0_nt25/``, etc. Each child must itself look like a run folder
(``repNN/.../summary.json``).
Usage:
python -m v4.scripts.analysis.compare_grid <grid_folder>
[--sort {test,val,name,reps}] [--reverse] [--csv]
Examples:
python -m v4.scripts.analysis.compare_grid \
v4/results/experiments/tri_v1/grid
python -m v4.scripts.analysis.compare_grid \
v4/results/experiments/tri_v1 --sort test
"""
from __future__ import annotations
import argparse
import csv
import sys
from pathlib import Path
from v4.scripts.analysis.summarize_run import summarise
def collect(grid_dir: Path) -> list[dict]:
rows: list[dict] = []
for child in sorted(grid_dir.iterdir()):
if not child.is_dir():
continue
s = summarise(child)
if s["n_reps"] == 0:
continue
rows.append({
"name": child.name,
"n": s["n_reps"],
"val_mean": s["val_mean"],
"val_std": s["val_std"],
"test_mean": s["test_mean"],
"test_std": s["test_std"],
})
return rows
def render_table(rows: list[dict]) -> str:
if not rows:
return "(no runs found)"
name_w = max(len("name"), max(len(r["name"]) for r in rows))
header = f"{'name':<{name_w}s} {'reps':>4s} {'val AUC':>17s} {'test AUC':>17s}"
sep = "-" * len(header)
lines = [header, sep]
for r in rows:
lines.append(
f"{r['name']:<{name_w}s} {r['n']:>4d} "
f"{r['val_mean']:.4f} ± {r['val_std']:.4f} "
f"{r['test_mean']:.4f} ± {r['test_std']:.4f}"
)
return "\n".join(lines)
def render_csv(rows: list[dict]) -> str:
buf = sys.stdout
w = csv.writer(buf)
w.writerow(["name", "reps", "val_mean", "val_std", "test_mean", "test_std"])
for r in rows:
w.writerow([r["name"], r["n"],
f"{r['val_mean']:.6f}", f"{r['val_std']:.6f}",
f"{r['test_mean']:.6f}", f"{r['test_std']:.6f}"])
return ""
_SORT_KEYS = {
"test": lambda r: r["test_mean"],
"val": lambda r: r["val_mean"],
"name": lambda r: r["name"],
"reps": lambda r: r["n"],
}
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("grid_dir", type=Path,
help="Folder whose immediate children are run folders")
ap.add_argument("--sort", choices=list(_SORT_KEYS),
default="test", help="Sort by which column (default: test)")
ap.add_argument("--reverse", action="store_true",
help="Reverse the default ordering")
ap.add_argument("--csv", action="store_true",
help="Emit CSV to stdout instead of a formatted table")
args = ap.parse_args()
if not args.grid_dir.is_dir():
raise SystemExit(f"Not a directory: {args.grid_dir}")
rows = collect(args.grid_dir)
descending = args.sort in {"test", "val", "reps"}
if args.reverse:
descending = not descending
rows.sort(key=_SORT_KEYS[args.sort], reverse=descending)
if args.csv:
render_csv(rows)
else:
print(f"Grid: {args.grid_dir} ({len(rows)} runs, sorted by {args.sort})\n")
print(render_table(rows))
if __name__ == "__main__":
main()
v4/scripts/analysis/inspect_embeddings.py
+98
View File
@@ -0,0 +1,98 @@
"""inspect_embeddings — per-dim statistics on a run's saved feature embeddings.
Loads features.h5 from a run folder (must have been produced with
``save_features: true`` in the config), computes per-dimension variance,
sparsity (fraction of |z| < threshold), and useful summary stats.
Helpful for diagnosing whether a fusion stage is collapsing dimensions to
near-zero — which would silently null out information when the bridge uses
a Hadamard product.
Usage:
python -m v4.scripts.analysis.inspect_embeddings <run_folder> [--phase NAME]
[--threshold 0.05]
Examples:
python -m v4.scripts.analysis.inspect_embeddings \
v4/results/experiments/tri_v1/baseline_tri/rep00 --phase nt
"""
from __future__ import annotations
import argparse
from pathlib import Path
import h5py
import numpy as np
def load_phase(features_path: Path, phase: str | None):
with h5py.File(features_path, "r") as f:
phases = list(f.keys())
if phase is None:
phase = phases[-1]
if phase not in phases:
raise SystemExit(
f"Phase {phase!r} not in {features_path} (available: {phases})"
)
g = f[phase]
z = g["z"][:] # (n_folds, n_samples, n_dim)
split = g["split"][:].astype(str) # (n_folds, n_samples)
y_true = g["y_true"][:]
return phase, z, split, y_true, phases
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("run_dir", type=Path, help="Folder containing features.h5 (under repNN)")
ap.add_argument("--phase", default=None,
help="Bridge phase name (e.g. nt, hb, cd_fuse). Default: last in file.")
ap.add_argument("--threshold", type=float, default=0.05,
help="|z| threshold for the sparsity count (default: 0.05)")
args = ap.parse_args()
feat = next(iter(args.run_dir.rglob("features.h5")), None)
if feat is None:
raise SystemExit(f"No features.h5 found under {args.run_dir}")
phase, z, split, _, all_phases = load_phase(feat, args.phase)
val_mask = (split == "val")
z_val = z[val_mask] # (n_val_total, n_dim)
z_train = z[(split == "train")]
n_dim = z_val.shape[-1]
print(f"Run: {args.run_dir}")
print(f"File: {feat}")
print(f"Phases: {all_phases}")
print(f"Phase: {phase} (z shape: {z.shape})")
print(f"Split sizes: train={len(z_train)} val={len(z_val)}")
print()
abs_z = np.abs(z_val)
per_dim_var = z_val.var(axis=0)
per_dim_abs = abs_z.mean(axis=0)
per_dim_max = abs_z.max(axis=0)
sparsity = (abs_z < args.threshold).mean(axis=0)
print(f"Aggregate stats over val embeddings (|z| < {args.threshold} = 'near-zero'):")
print(f" global mean(|z|): {abs_z.mean():.4f}")
print(f" global var(z): {z_val.var():.4f}")
print(f" fraction near-zero (global): {(abs_z < args.threshold).mean():.4f}")
print()
print(f"Per-dimension summary ({n_dim} dims):")
print(f" variance: min={per_dim_var.min():.4f} med={np.median(per_dim_var):.4f} max={per_dim_var.max():.4f}")
print(f" mean |z|: min={per_dim_abs.min():.4f} med={np.median(per_dim_abs):.4f} max={per_dim_abs.max():.4f}")
print(f" near-zero rate: min={sparsity.min():.4f} med={np.median(sparsity):.4f} max={sparsity.max():.4f}")
# Dead dimensions: high near-zero rate
dead = np.where(sparsity > 0.9)[0]
print(f" 'dead' dims (>90% near-zero): {len(dead)}/{n_dim} "
f"{('idx: ' + str(dead.tolist())) if 0 < len(dead) <= 20 else ''}")
weak = np.where(per_dim_var < 1e-4)[0]
print(f" 'weak' dims (var < 1e-4): {len(weak)}/{n_dim} "
f"{('idx: ' + str(weak.tolist())) if 0 < len(weak) <= 20 else ''}")
if __name__ == "__main__":
main()
v4/scripts/analysis/summarize_run.py
+132
View File
@@ -0,0 +1,132 @@
"""summarize_run — print cross-rep stats for one v4 run folder.
A "run" is a folder like ``v4/results/experiments/tri_v1/grid/bcd75_cw1_nt25/``
containing ``rep00/``, ``rep01/``, ... — each with a per-rep ``summary.json``
under any ``out_dir_tags`` subdir (typically ``binary/`` or ``binary/ntower/``).
Usage:
python -m v4.scripts.analysis.summarize_run <run_folder> [--per-rep] [--json]
Examples:
python -m v4.scripts.analysis.summarize_run \
v4/results/experiments/tri_v1/grid/bcd75_cw1_nt25
python -m v4.scripts.analysis.summarize_run \
v4/results/experiments/tri_v1/baseline_tri --per-rep
"""
from __future__ import annotations
import argparse
import json
import re
from pathlib import Path
import numpy as np
def find_rep_summaries(run_dir: Path) -> list[tuple[int, Path]]:
"""Return [(rep_idx, summary_path), ...] sorted by rep_idx."""
out: list[tuple[int, Path]] = []
for rep_dir in sorted(run_dir.glob("rep*")):
if not rep_dir.is_dir():
continue
m = re.match(r"rep(\d+)$", rep_dir.name)
if not m:
continue
summary = next(iter(rep_dir.rglob("summary.json")), None)
if summary is not None:
out.append((int(m.group(1)), summary))
return out
def load_rep(summary_path: Path) -> dict:
"""Extract the fields we summarise from one rep's summary.json."""
d = json.loads(summary_path.read_text())
return {
"val_mean": float(d.get("mean_val_auc", float("nan"))),
"val_std": float(d.get("std_val_auc", float("nan"))),
"test_mean": float(d.get("mean_test_auc", float("nan"))),
"test_std": float(d.get("std_test_auc", float("nan"))),
"elapsed_s": float(d.get("elapsed_s", float("nan"))),
"eval_stage": d.get("eval_stage", "?"),
}
def summarise(run_dir: Path) -> dict:
reps = find_rep_summaries(run_dir)
if not reps:
return {"run": str(run_dir), "n_reps": 0, "reps": []}
rows = [(idx, load_rep(p)) for idx, p in reps]
val = np.array([r[1]["val_mean"] for r in rows])
test = np.array([r[1]["test_mean"] for r in rows])
elaps = np.array([r[1]["elapsed_s"] for r in rows])
out = {
"run": str(run_dir),
"n_reps": len(rows),
"eval_stage": rows[0][1]["eval_stage"],
"val_mean": float(np.mean(val)),
"val_std": float(np.std(val)),
"val_min": float(np.min(val)),
"val_max": float(np.max(val)),
"test_mean": float(np.mean(test)),
"test_std": float(np.std(test)),
"test_min": float(np.min(test)),
"test_max": float(np.max(test)),
"elapsed_total_s": float(np.sum(elaps)) if not np.isnan(elaps).any() else None,
"reps": [
{"rep": idx, **info} for idx, info in rows
],
}
return out
def render(s: dict, per_rep: bool = False) -> str:
if s["n_reps"] == 0:
return f"Run: {s['run']}\n no reps with summary.json found."
lines = [
f"Run: {s['run']}",
f"Reps: {s['n_reps']} (eval_stage={s['eval_stage']})",
f"Val AUC: {s['val_mean']:.4f} ± {s['val_std']:.4f} "
f"[min={s['val_min']:.4f} max={s['val_max']:.4f}]",
f"Test AUC: {s['test_mean']:.4f} ± {s['test_std']:.4f} "
f"[min={s['test_min']:.4f} max={s['test_max']:.4f}]",
]
if s.get("elapsed_total_s") is not None:
h = s["elapsed_total_s"] / 3600
lines.append(f"Compute: {s['elapsed_total_s']:.0f} s total ({h:.1f} h)")
if per_rep:
lines.append("")
lines.append("Per-rep breakdown:")
lines.append(f" {'rep':>4s} {'val':>7s} {'test':>7s} {'elapsed':>7s}")
for r in s["reps"]:
elapsed = (f"{r['elapsed_s']:.0f}s" if not np.isnan(r['elapsed_s']) else "-")
lines.append(
f" {r['rep']:>4d} "
f"{r['val_mean']:>7.4f} "
f"{r['test_mean']:>7.4f} "
f"{elapsed:>7s}"
)
return "\n".join(lines)
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("run_dir", type=Path, help="Path to a run folder (contains repNN/ subdirs)")
ap.add_argument("--per-rep", action="store_true", help="Print one line per rep")
ap.add_argument("--json", action="store_true", help="Emit JSON instead of formatted text")
args = ap.parse_args()
if not args.run_dir.is_dir():
raise SystemExit(f"Not a directory: {args.run_dir}")
s = summarise(args.run_dir)
if args.json:
print(json.dumps(s, indent=2))
else:
print(render(s, per_rep=args.per_rep))
if __name__ == "__main__":
main()
v4/scripts/experiments/tri_v1/baseline_tri_features.json
+10
View File
@@ -0,0 +1,10 @@
[
{
"_note": "Single rep of tritower default with save_features=true so we can inspect nt-stage embeddings (Hadamard-product collapse hypothesis).",
"run_name": "experiments/tri_v1/baseline_tri_features",
"reps": 1,
"overrides": {
"save_features": true
}
}
]
v4/scripts/experiments/tri_v1/cd_solo_geom_warm_sweep.json
+13
View File
@@ -0,0 +1,13 @@
[
{ "_note": "cd-solo + UNet geom-vector inject; cd_warm sweep. 3 reps each.",
"run_name": "experiments/tri_v1/cd_solo_geom/warm00", "reps": 3,
"stage_overrides": { "cd_warm": { "epochs": 0 } } },
{ "run_name": "experiments/tri_v1/cd_solo_geom/warm05", "reps": 3,
"stage_overrides": { "cd_warm": { "epochs": 5 } } },
{ "run_name": "experiments/tri_v1/cd_solo_geom/warm10", "reps": 3,
"stage_overrides": { "cd_warm": { "epochs": 10 } } },
{ "run_name": "experiments/tri_v1/cd_solo_geom/warm20", "reps": 3,
"stage_overrides": { "cd_warm": { "epochs": 20 } } },
{ "run_name": "experiments/tri_v1/cd_solo_geom/warm40", "reps": 3,
"stage_overrides": { "cd_warm": { "epochs": 40 } } }
]
v4/scripts/experiments/tri_v1/cd_solo_warm_sweep.json
+13
View File
@@ -0,0 +1,13 @@
[
{ "_note": "cd-solo, no geom; cd_warm sweep. 3 reps each. Companion to cd_solo_geom sweep.",
"run_name": "experiments/tri_v1/cd_solo/warm00", "reps": 3,
"stage_overrides": { "cd_warm": { "epochs": 0 } } },
{ "run_name": "experiments/tri_v1/cd_solo/warm05", "reps": 3,
"stage_overrides": { "cd_warm": { "epochs": 5 } } },
{ "run_name": "experiments/tri_v1/cd_solo/warm10", "reps": 3,
"stage_overrides": { "cd_warm": { "epochs": 10 } } },
{ "run_name": "experiments/tri_v1/cd_solo/warm20", "reps": 3,
"stage_overrides": { "cd_warm": { "epochs": 20 } } },
{ "run_name": "experiments/tri_v1/cd_solo/warm40", "reps": 3,
"stage_overrides": { "cd_warm": { "epochs": 40 } } }
]
v4/scripts/experiments/tri_v1/promote_bcd75_cw1_nt25.json
+16
View File
@@ -0,0 +1,16 @@
[
{
"_note": "Promote grid winner bcd75_cw1_nt25 (3-rep test AUC 0.910) to 10 reps. First 3 reps will be skipped (already on disk).",
"run_name": "experiments/tri_v1/grid/bcd75_cw1_nt25",
"reps": 10,
"overrides": {
"training": {
"bcd_prob": 0.75,
"class_weighted": true
}
},
"stage_overrides": {
"nt": { "epochs": 25 }
}
}
]