diff --git a/results/poster/papila_binary_roc_comparison.json b/results/poster/papila_binary_roc_comparison.json new file mode 100644 index 0000000..62d0448 --- /dev/null +++ b/results/poster/papila_binary_roc_comparison.json @@ -0,0 +1,61 @@ +{ + "figure": "/home/rpotter/hypertower/results/poster/papila_binary_roc_comparison.png", + "curves": [ + { + "label": "Clinical Data Only", + "source_kind": "legacy_npy", + "mode_dir": "analysis_data/pipeline_mdonly_500ep/binary/single", + "run": null, + "tower_path": null, + "score_col": null, + "probs_stem": "probs_classic", + "auc_mean": 0.7311688311688311, + "auc_std": 0.08461127279353979, + "auc_pooled": 0.6812987012987013, + "fold_count": 5, + "n_total": 400 + }, + { + "label": "Image Only", + "source_kind": "v3_csv", + "mode_dir": null, + "run": "phase2/imageonly_resnet50_proper", + "tower_path": "binary/single", + "score_col": "prob_img_c1", + "probs_stem": null, + "auc_mean": 0.819264705882353, + "auc_std": 0.0740851608018641, + "auc_pooled": 0.8099584558823529, + "fold_count": 50, + "n_total": 4200 + }, + { + "label": "Single Fusion", + "source_kind": "v3_csv", + "mode_dir": null, + "run": "phase5/single_fused", + "tower_path": "binary/single", + "score_col": "prob_fused_c1", + "probs_stem": null, + "auc_mean": 0.8506801470588234, + "auc_std": 0.07392522372615995, + "auc_pooled": 0.8402091911764706, + "fold_count": 50, + "n_total": 4200 + }, + { + "label": "Ensemble Fusion", + "source_kind": "v3_csv", + "mode_dir": null, + "run": "phase5/logit_mlp_head", + "tower_path": "binary/ensemble", + "score_col": "prob_fused_c1", + "probs_stem": null, + "auc_mean": 0.8988235294117648, + "auc_std": 0.06435919826554772, + "auc_pooled": 0.8855705882352942, + "fold_count": 50, + "n_total": 2100 + } + ] +} \ No newline at end of file diff --git a/results/poster/papila_binary_roc_comparison.png b/results/poster/papila_binary_roc_comparison.png new file mode 100644 index 0000000..bd4c310 Binary files /dev/null and b/results/poster/papila_binary_roc_comparison.png differ diff --git a/v3/figures/explainability/poster_model_comparison.png b/v3/figures/explainability/poster_model_comparison.png new file mode 100644 index 0000000..b24246e Binary files /dev/null and b/v3/figures/explainability/poster_model_comparison.png differ diff --git a/v3/scripts/output_analysis/explainability/poster_model_comparison.py b/v3/scripts/output_analysis/explainability/poster_model_comparison.py new file mode 100644 index 0000000..54f8ef8 --- /dev/null +++ b/v3/scripts/output_analysis/explainability/poster_model_comparison.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +""" +Poster-facing model comparison figure in the style of head_comparison.png. + +Compares: + - MD Only + - Image Only + - Ensemble Fusion + +using the same run sources as the poster ROC comparison script. +""" +from __future__ import annotations + +import argparse +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + +from v3.scripts.output_analysis.plot_poster_roc_comparison import ( + DEFAULT_CURVES, + REPO_ROOT, + build_curve_summary, + load_curve_predictions, +) + + +DEFAULT_OUT = REPO_ROOT / "v3" / "figures" / "explainability" / "poster_model_comparison.png" +MODEL_ORDER = ["Clinical Data Only", "Image Only", "Ensemble Fusion"] + +C_NORMAL = "#4c72b0" +C_GLAUCOMA = "#c44e52" +BG = "#e8e8e8" + + +def _select_specs(): + selected = [] + for label in MODEL_ORDER: + matches = [spec for spec in DEFAULT_CURVES if spec.label == label] + if not matches: + raise ValueError(f"Could not find default poster curve for '{label}'") + selected.append(matches[0]) + return selected + + +def make_model_comparison(out_path: Path) -> None: + specs = _select_specs() + rng = np.random.default_rng(42) + + fig, axes = plt.subplots(1, len(specs), figsize=(11, 4.5), sharey=True) + if len(specs) == 1: + axes = [axes] + fig.patch.set_facecolor(BG) + fig.suptitle( + "Model Comparison — P(Glaucoma) by True Class", + fontsize=12, + fontweight="bold", + ) + + for ax, spec in zip(axes, specs): + df, score_col = load_curve_predictions(spec) + summary = build_curve_summary(df, score_col) + + ax.set_facecolor(BG) + data_by_class = [df.loc[df["y_true"] == cls, score_col].values for cls in [0, 1]] + + vp = ax.violinplot( + data_by_class, + positions=[0, 1], + widths=0.6, + showmedians=True, + showextrema=False, + ) + for body, color in zip(vp["bodies"], [C_NORMAL, C_GLAUCOMA]): + body.set_facecolor(color) + body.set_alpha(0.35) + vp["cmedians"].set_color("#222") + vp["cmedians"].set_linewidth(2) + + for cls, color in zip([0, 1], [C_NORMAL, C_GLAUCOMA]): + vals = data_by_class[cls] + jitter = rng.uniform(-0.12, 0.12, len(vals)) + ax.scatter( + cls + jitter, + vals, + color=color, + s=4, + alpha=0.30, + linewidths=0, + zorder=3, + ) + + ax.axhline(0.5, color="#888", lw=1.0, ls="--", alpha=0.6) + ax.set_xticks([0, 1]) + ax.set_xticklabels(["Normal", "Glaucoma"], fontsize=9) + ax.set_title(spec.label, fontsize=10, fontweight="bold", color=spec.color) + ax.set_ylim(-0.05, 1.05) + ax.grid(axis="y", alpha=0.3) + if ax is axes[0]: + ax.set_ylabel("Predicted P(Glaucoma)", fontsize=10) + + ax.text( + 0.97, + 0.04, + f"AUC = {summary['auc_mean']:.3f} ± {summary['auc_std']:.3f}", + transform=ax.transAxes, + ha="right", + va="bottom", + fontsize=9, + color="#333", + bbox=dict(facecolor="white", alpha=0.65, edgecolor="none", pad=2), + ) + + fig.tight_layout() + out_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(out_path, dpi=180, bbox_inches="tight") + plt.close(fig) + print(f"Saved: {out_path}") + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--out", type=Path, default=DEFAULT_OUT) + args = ap.parse_args() + make_model_comparison(args.out) + + +if __name__ == "__main__": + main() diff --git a/v3/scripts/output_analysis/plot_poster_roc_comparison.py b/v3/scripts/output_analysis/plot_poster_roc_comparison.py new file mode 100644 index 0000000..0bf0fed --- /dev/null +++ b/v3/scripts/output_analysis/plot_poster_roc_comparison.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +""" +Poster-facing ROC comparison for PAPILA binary glaucoma classification. + +Builds one combined ROC figure with four curves: + - MD Only + - Image Only + - Single Fusion + - Ensemble Fusion + +The default setup mixes two artifact layouts: + 1. Dedicated unimodal runs saved under analysis_data/.../fold*/{y_true,probs}.npy + 2. Repeated CV fusion runs saved under v3/results/.../rep*/.../predictions_test.csv + +Update DEFAULT_CURVES below if you want different source runs or labels. +""" +from __future__ import annotations + +import argparse +import json +from dataclasses import dataclass +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from sklearn.metrics import roc_auc_score, roc_curve + + +REPO_ROOT = Path(__file__).resolve().parents[3] +RESULTS_ROOT = REPO_ROOT / "v3" / "results" +DEFAULT_OUT = REPO_ROOT / "results" / "poster" / "papila_binary_roc_comparison.png" + + +@dataclass(frozen=True) +class CurveSpec: + label: str + source_kind: str + color: str + mode_dir: str | None = None + probs_stem: str | None = None + run: str | None = None + tower_path: str | None = None + score_col: str | None = None + + +DEFAULT_CURVES = [ + CurveSpec( + label="Clinical Data Only", + source_kind="legacy_npy", + color="#4c72b0", + mode_dir="analysis_data/pipeline_mdonly_500ep/binary/single", + probs_stem="probs_classic", + ), + CurveSpec( + label="Image Only", + source_kind="v3_csv", + color="#dd8452", + run="phase2/imageonly_resnet50_proper", + tower_path="binary/single", + score_col="prob_img_c1", + ), + CurveSpec( + label="Single Fusion", + source_kind="v3_csv", + color="#55a868", + run="phase5/single_fused", + tower_path="binary/single", + score_col="prob_fused_c1", + ), + CurveSpec( + label="Ensemble Fusion", + source_kind="v3_csv", + color="#c44e52", + run="phase5/logit_mlp_head", + tower_path="binary/ensemble", + score_col="prob_fused_c1", + ), +] + + +def load_v3_csv_predictions(run: str, tower_path: str, score_col: str) -> pd.DataFrame: + run_dir = RESULTS_ROOT / run + rows: list[pd.DataFrame] = [] + for rep_dir in sorted(run_dir.glob("rep*")): + mode_dir = rep_dir / tower_path + if not mode_dir.exists(): + continue + for fold_dir in sorted(mode_dir.glob("fold[0-9]")): + csv_path = fold_dir / "predictions_test.csv" + if not csv_path.exists(): + continue + df = pd.read_csv(csv_path, usecols=["y_true", score_col]) + df["rep"] = rep_dir.name + df["fold"] = fold_dir.name + rows.append(df) + if not rows: + raise FileNotFoundError(f"No predictions found under {run_dir}/{tower_path}") + return pd.concat(rows, ignore_index=True) + + +def load_legacy_npy_predictions(mode_dir: str, probs_stem: str) -> pd.DataFrame: + root = REPO_ROOT / mode_dir + rows: list[pd.DataFrame] = [] + for fold_dir in sorted(root.glob("fold[0-9]")): + y_path = fold_dir / "y_true.npy" + p_path = fold_dir / f"{probs_stem}.npy" + if not y_path.exists() or not p_path.exists(): + continue + y_true = np.load(y_path) + probs = np.load(p_path) + if probs.ndim == 2: + if probs.shape[1] < 2: + raise ValueError(f"Expected 2-class probs in {p_path}") + scores = probs[:, 1] + else: + scores = probs + rows.append( + pd.DataFrame( + { + "y_true": y_true, + "score": scores, + "rep": "rep00", + "fold": fold_dir.name, + } + ) + ) + if not rows: + raise FileNotFoundError(f"No fold artifacts found under {root}") + return pd.concat(rows, ignore_index=True) + + +def load_curve_predictions(spec: CurveSpec) -> tuple[pd.DataFrame, str]: + if spec.source_kind == "v3_csv": + if spec.run is None or spec.tower_path is None or spec.score_col is None: + raise ValueError(f"Incomplete v3_csv spec: {spec}") + return load_v3_csv_predictions(spec.run, spec.tower_path, spec.score_col), spec.score_col + if spec.source_kind == "legacy_npy": + if spec.mode_dir is None or spec.probs_stem is None: + raise ValueError(f"Incomplete legacy_npy spec: {spec}") + return load_legacy_npy_predictions(spec.mode_dir, spec.probs_stem), "score" + raise ValueError(f"Unknown source_kind: {spec.source_kind}") + + +def build_curve_summary(df: pd.DataFrame, score_col: str) -> dict[str, object]: + if score_col not in df.columns: + raise KeyError(f"Missing score column '{score_col}'") + + mean_fpr = np.linspace(0.0, 1.0, 501) + tprs: list[np.ndarray] = [] + aucs: list[float] = [] + fold_count = 0 + + for (_, _), fold_df in df.groupby(["rep", "fold"], sort=True): + y_true = fold_df["y_true"].to_numpy() + scores = fold_df[score_col].to_numpy() + if len(np.unique(y_true)) < 2: + continue + fpr, tpr, _ = roc_curve(y_true, scores) + interp_tpr = np.interp(mean_fpr, fpr, tpr) + interp_tpr[0] = 0.0 + interp_tpr[-1] = 1.0 + tprs.append(interp_tpr) + aucs.append(float(roc_auc_score(y_true, scores))) + fold_count += 1 + + if not tprs: + raise ValueError(f"No valid binary folds found for '{score_col}'") + + tpr_arr = np.vstack(tprs) + y_all = df["y_true"].to_numpy() + s_all = df[score_col].to_numpy() + + return { + "fpr": mean_fpr, + "tpr_mean": tpr_arr.mean(axis=0), + "tpr_std": tpr_arr.std(axis=0), + "auc_mean": float(np.mean(aucs)), + "auc_std": float(np.std(aucs)), + "auc_pooled": float(roc_auc_score(y_all, s_all)), + "fold_count": fold_count, + "n_total": int(len(df)), + } + + +def plot_curves(curve_summaries: list[tuple[CurveSpec, dict[str, object]]], out_path: Path) -> None: + out_path.parent.mkdir(parents=True, exist_ok=True) + + fig, ax = plt.subplots(figsize=(8.8, 7.0)) + ax.plot([0, 1], [0, 1], linestyle="--", linewidth=1, color="0.5", alpha=0.8) + + for spec, summary in curve_summaries: + fpr = np.asarray(summary["fpr"]) + tpr_mean = np.asarray(summary["tpr_mean"]) + tpr_std = np.asarray(summary["tpr_std"]) + auc_mean = float(summary["auc_mean"]) + auc_std = float(summary["auc_std"]) + + ax.plot( + fpr, + tpr_mean, + color=spec.color, + linewidth=2.4, + label=f"{spec.label} (AUC {auc_mean:.3f} ± {auc_std:.3f})", + ) + ax.fill_between( + fpr, + np.clip(tpr_mean - tpr_std, 0, 1), + np.clip(tpr_mean + tpr_std, 0, 1), + color=spec.color, + alpha=0.12, + ) + + ax.set_xlim(-0.01, 1.01) + ax.set_ylim(-0.01, 1.01) + ax.set_xlabel("False Positive Rate") + ax.set_ylabel("True Positive Rate") + ax.set_title("PAPILA Binary ROC Comparison") + ax.grid(alpha=0.25) + ax.legend(loc="lower right", frameon=True) + + fig.tight_layout() + fig.savefig(out_path, dpi=200, bbox_inches="tight") + plt.close(fig) + + +def write_summary(curve_summaries: list[tuple[CurveSpec, dict[str, object]]], out_path: Path) -> None: + summary_path = out_path.with_suffix(".json") + payload = { + "figure": str(out_path), + "curves": [ + { + "label": spec.label, + "source_kind": spec.source_kind, + "mode_dir": spec.mode_dir, + "run": spec.run, + "tower_path": spec.tower_path, + "score_col": spec.score_col, + "probs_stem": spec.probs_stem, + "auc_mean": float(summary["auc_mean"]), + "auc_std": float(summary["auc_std"]), + "auc_pooled": float(summary["auc_pooled"]), + "fold_count": int(summary["fold_count"]), + "n_total": int(summary["n_total"]), + } + for spec, summary in curve_summaries + ], + } + summary_path.write_text(json.dumps(payload, indent=2)) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument( + "--output", + type=Path, + default=DEFAULT_OUT, + help=f"Output PNG path (default: {DEFAULT_OUT})", + ) + args = ap.parse_args() + + curve_summaries: list[tuple[CurveSpec, dict[str, object]]] = [] + for spec in DEFAULT_CURVES: + df, score_col = load_curve_predictions(spec) + summary = build_curve_summary(df, score_col) + curve_summaries.append((spec, summary)) + print( + f"{spec.label:16s} " + f"AUC={summary['auc_mean']:.4f} ± {summary['auc_std']:.4f} " + f"pooled={summary['auc_pooled']:.4f} " + f"folds={summary['fold_count']}" + ) + + plot_curves(curve_summaries, args.output) + write_summary(curve_summaries, args.output) + print(f"Saved figure: {args.output}") + print(f"Saved summary: {args.output.with_suffix('.json')}") + + +if __name__ == "__main__": + main() diff --git a/v4/configs/clinical_solo.json b/v4/configs/clinical_solo.json new file mode 100644 index 0000000..8a179d2 --- /dev/null +++ b/v4/configs/clinical_solo.json @@ -0,0 +1,78 @@ +{ + "_notes": [ + "Diagnostic: cd tower alone, no geometry injection.", + "Pair to clinical_solo_geom_unet — same architecture but with geom_dim=0", + "and no img EPC supplier." + ], + "run_name": "v4/clinical_solo", + "num_classes": 2, + "label_filter": [0, 1], + "split_identity_level": 1, + "eval_stage": "cd_fuse", + "save_predictions": true, + "save_features": true, + "seed": 1234, + "folds": 5, + "fold_seed": 100, + "output_root": "v4/results", + "out_dir_tags": ["binary"], + + "data": { + "module": "v4.classes.profiles.v4papila", + "args": { + "image_dir": "Papila/FundusImages", + "clinical_dir": "Papila/ClinicalData", + "label_col": "Diagnosis", + "iop_corr_method": "ratio", + "iop_drop_raw": true, + "exclude_cols": ["Axial_Length"], + "in_memory_cache": false + } + }, + + "towers": [ + { + "name": "cd", + "module": "v4.classes.towers.clinical_tower", + "class": "ClinicalEncoder", + "data_source": "matrix", + "args": { + "hidden_dim": 128 + } + } + ], + + "stages": [ + { + "name": "cd_warm", + "type": "warm", + "tower": "cd", + "head_name": "cd_aux", + "level": "eye", + "epochs": 40 + }, + { + "name": "cd_aux", + "type": "head", + "input": "cd", + "train_with": "cd_fuse" + }, + { + "name": "cd_fuse", + "type": "fusion", + "module": "v4.classes.bridges.mono_bridge", + "class": "MonoBridge", + "inputs": ["cd"], + "level": "eye", + "epochs": 36, + "train_towers": true, + "args": { "use_ln": false } + } + ], + + "training": { + "lr": 1e-4, + "batch_size": 16, + "tune_binary_threshold": true + } +} diff --git a/v4/configs/clinical_solo_geom_unet.json b/v4/configs/clinical_solo_geom_unet.json new file mode 100644 index 0000000..54dc3c0 --- /dev/null +++ b/v4/configs/clinical_solo_geom_unet.json @@ -0,0 +1,103 @@ +{ + "_notes": [ + "Diagnostic: cd tower alone, with UNet-derived 5-feature CDR vector injection.", + "img tower is present only as an EPC supplier — its early_pass runs the UNet", + "fine-tune+inference pipeline and publishes geometry_vectors, but no stage", + "uses its embedding so its CNN body sits idle.", + "cd consumes the vectors via epc_requests and trains under MonoBridge." + ], + "run_name": "v4/clinical_solo_geom_unet", + "num_classes": 2, + "label_filter": [0, 1], + "split_identity_level": 1, + "eval_stage": "cd_fuse", + "save_predictions": true, + "save_features": true, + "seed": 1234, + "folds": 5, + "fold_seed": 100, + "output_root": "v4/results", + "out_dir_tags": ["binary"], + + "data": { + "module": "v4.classes.profiles.v4papila", + "args": { + "image_dir": "Papila/FundusImages", + "clinical_dir": "Papila/ClinicalData", + "label_col": "Diagnosis", + "iop_corr_method": "ratio", + "iop_drop_raw": true, + "exclude_cols": ["Axial_Length"], + "in_memory_cache": false + } + }, + + "towers": [ + { + "name": "img", + "module": "v4.classes.towers.image_tower", + "class": "ImageEncoder", + "data_source": "image", + "epc_supplies": ["geometry_vectors"], + "args": { + "backbone": "refugelike", + "freeze_ratio": 1.0, + "augment": false, + "geometry_source": "unet", + "weights_path": "models/v2/refuge/segmentation/per_image/best.pt", + "contour_dir": "Papila/ExpertsSegmentations/Contours", + "unet_size": 512, + "normalize": "per_image", + "threshold": 0.5, + "finetune_epochs": 10, + "finetune_lr": 1e-5, + "finetune_batch_size": 4 + } + }, + { + "name": "cd", + "module": "v4.classes.towers.clinical_tower", + "class": "ClinicalEncoder", + "data_source": "matrix", + "epc_requests": ["geometry_vectors"], + "args": { + "hidden_dim": 128, + "geom_dim": 5 + } + } + ], + + "stages": [ + { + "name": "cd_warm", + "type": "warm", + "tower": "cd", + "head_name": "cd_aux", + "level": "eye", + "epochs": 40 + }, + { + "name": "cd_aux", + "type": "head", + "input": "cd", + "train_with": "cd_fuse" + }, + { + "name": "cd_fuse", + "type": "fusion", + "module": "v4.classes.bridges.mono_bridge", + "class": "MonoBridge", + "inputs": ["cd"], + "level": "eye", + "epochs": 36, + "train_towers": true, + "args": { "use_ln": false } + } + ], + + "training": { + "lr": 1e-4, + "batch_size": 16, + "tune_binary_threshold": true + } +} diff --git a/v4/distributed/batch_dispatch.py b/v4/distributed/batch_dispatch.py index d4af6ff..f340f94 100644 --- a/v4/distributed/batch_dispatch.py +++ b/v4/distributed/batch_dispatch.py @@ -169,7 +169,7 @@ def dispatch_batch( server_cfg_path = str(rel_path) job_body = { - "run_name": run_name, + "run_name": cfg["run_name"], # per-rep, e.g. ".../rep05" "module": "v4.classes.v4_hypertower", "args": ["--config", server_cfg_path], "output_dir": output_root, diff --git a/v4/distributed/jobs.db b/v4/distributed/jobs.db index 107ec39..5f5bc10 100644 Binary files a/v4/distributed/jobs.db and b/v4/distributed/jobs.db differ diff --git a/v4/scripts/__init__.py b/v4/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/v4/scripts/analysis/__init__.py b/v4/scripts/analysis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/v4/scripts/analysis/compare_grid.py b/v4/scripts/analysis/compare_grid.py new file mode 100644 index 0000000..13038d4 --- /dev/null +++ b/v4/scripts/analysis/compare_grid.py @@ -0,0 +1,112 @@ +"""compare_grid — print a ranked comparison table for a folder of v4 runs. + +A "grid" folder is one whose immediate children are individual run folders, e.g. +``v4/results/experiments/tri_v1/grid/`` containing ``bcd35_cw0_nt15/``, +``bcd35_cw0_nt25/``, etc. Each child must itself look like a run folder +(``repNN/.../summary.json``). + +Usage: + python -m v4.scripts.analysis.compare_grid + [--sort {test,val,name,reps}] [--reverse] [--csv] + +Examples: + python -m v4.scripts.analysis.compare_grid \ + v4/results/experiments/tri_v1/grid + python -m v4.scripts.analysis.compare_grid \ + v4/results/experiments/tri_v1 --sort test +""" +from __future__ import annotations + +import argparse +import csv +import sys +from pathlib import Path + +from v4.scripts.analysis.summarize_run import summarise + + +def collect(grid_dir: Path) -> list[dict]: + rows: list[dict] = [] + for child in sorted(grid_dir.iterdir()): + if not child.is_dir(): + continue + s = summarise(child) + if s["n_reps"] == 0: + continue + rows.append({ + "name": child.name, + "n": s["n_reps"], + "val_mean": s["val_mean"], + "val_std": s["val_std"], + "test_mean": s["test_mean"], + "test_std": s["test_std"], + }) + return rows + + +def render_table(rows: list[dict]) -> str: + if not rows: + return "(no runs found)" + name_w = max(len("name"), max(len(r["name"]) for r in rows)) + header = f"{'name':<{name_w}s} {'reps':>4s} {'val AUC':>17s} {'test AUC':>17s}" + sep = "-" * len(header) + lines = [header, sep] + for r in rows: + lines.append( + f"{r['name']:<{name_w}s} {r['n']:>4d} " + f"{r['val_mean']:.4f} ± {r['val_std']:.4f} " + f"{r['test_mean']:.4f} ± {r['test_std']:.4f}" + ) + return "\n".join(lines) + + +def render_csv(rows: list[dict]) -> str: + buf = sys.stdout + w = csv.writer(buf) + w.writerow(["name", "reps", "val_mean", "val_std", "test_mean", "test_std"]) + for r in rows: + w.writerow([r["name"], r["n"], + f"{r['val_mean']:.6f}", f"{r['val_std']:.6f}", + f"{r['test_mean']:.6f}", f"{r['test_std']:.6f}"]) + return "" + + +_SORT_KEYS = { + "test": lambda r: r["test_mean"], + "val": lambda r: r["val_mean"], + "name": lambda r: r["name"], + "reps": lambda r: r["n"], +} + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("grid_dir", type=Path, + help="Folder whose immediate children are run folders") + ap.add_argument("--sort", choices=list(_SORT_KEYS), + default="test", help="Sort by which column (default: test)") + ap.add_argument("--reverse", action="store_true", + help="Reverse the default ordering") + ap.add_argument("--csv", action="store_true", + help="Emit CSV to stdout instead of a formatted table") + args = ap.parse_args() + + if not args.grid_dir.is_dir(): + raise SystemExit(f"Not a directory: {args.grid_dir}") + + rows = collect(args.grid_dir) + descending = args.sort in {"test", "val", "reps"} + if args.reverse: + descending = not descending + rows.sort(key=_SORT_KEYS[args.sort], reverse=descending) + + if args.csv: + render_csv(rows) + else: + print(f"Grid: {args.grid_dir} ({len(rows)} runs, sorted by {args.sort})\n") + print(render_table(rows)) + + +if __name__ == "__main__": + main() diff --git a/v4/scripts/analysis/inspect_embeddings.py b/v4/scripts/analysis/inspect_embeddings.py new file mode 100644 index 0000000..49dc494 --- /dev/null +++ b/v4/scripts/analysis/inspect_embeddings.py @@ -0,0 +1,98 @@ +"""inspect_embeddings — per-dim statistics on a run's saved feature embeddings. + +Loads features.h5 from a run folder (must have been produced with +``save_features: true`` in the config), computes per-dimension variance, +sparsity (fraction of |z| < threshold), and useful summary stats. + +Helpful for diagnosing whether a fusion stage is collapsing dimensions to +near-zero — which would silently null out information when the bridge uses +a Hadamard product. + +Usage: + python -m v4.scripts.analysis.inspect_embeddings [--phase NAME] + [--threshold 0.05] + +Examples: + python -m v4.scripts.analysis.inspect_embeddings \ + v4/results/experiments/tri_v1/baseline_tri/rep00 --phase nt +""" +from __future__ import annotations + +import argparse +from pathlib import Path + +import h5py +import numpy as np + + +def load_phase(features_path: Path, phase: str | None): + with h5py.File(features_path, "r") as f: + phases = list(f.keys()) + if phase is None: + phase = phases[-1] + if phase not in phases: + raise SystemExit( + f"Phase {phase!r} not in {features_path} (available: {phases})" + ) + g = f[phase] + z = g["z"][:] # (n_folds, n_samples, n_dim) + split = g["split"][:].astype(str) # (n_folds, n_samples) + y_true = g["y_true"][:] + return phase, z, split, y_true, phases + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("run_dir", type=Path, help="Folder containing features.h5 (under repNN)") + ap.add_argument("--phase", default=None, + help="Bridge phase name (e.g. nt, hb, cd_fuse). Default: last in file.") + ap.add_argument("--threshold", type=float, default=0.05, + help="|z| threshold for the sparsity count (default: 0.05)") + args = ap.parse_args() + + feat = next(iter(args.run_dir.rglob("features.h5")), None) + if feat is None: + raise SystemExit(f"No features.h5 found under {args.run_dir}") + + phase, z, split, _, all_phases = load_phase(feat, args.phase) + + val_mask = (split == "val") + z_val = z[val_mask] # (n_val_total, n_dim) + z_train = z[(split == "train")] + n_dim = z_val.shape[-1] + + print(f"Run: {args.run_dir}") + print(f"File: {feat}") + print(f"Phases: {all_phases}") + print(f"Phase: {phase} (z shape: {z.shape})") + print(f"Split sizes: train={len(z_train)} val={len(z_val)}") + print() + + abs_z = np.abs(z_val) + per_dim_var = z_val.var(axis=0) + per_dim_abs = abs_z.mean(axis=0) + per_dim_max = abs_z.max(axis=0) + sparsity = (abs_z < args.threshold).mean(axis=0) + + print(f"Aggregate stats over val embeddings (|z| < {args.threshold} = 'near-zero'):") + print(f" global mean(|z|): {abs_z.mean():.4f}") + print(f" global var(z): {z_val.var():.4f}") + print(f" fraction near-zero (global): {(abs_z < args.threshold).mean():.4f}") + print() + print(f"Per-dimension summary ({n_dim} dims):") + print(f" variance: min={per_dim_var.min():.4f} med={np.median(per_dim_var):.4f} max={per_dim_var.max():.4f}") + print(f" mean |z|: min={per_dim_abs.min():.4f} med={np.median(per_dim_abs):.4f} max={per_dim_abs.max():.4f}") + print(f" near-zero rate: min={sparsity.min():.4f} med={np.median(sparsity):.4f} max={sparsity.max():.4f}") + + # Dead dimensions: high near-zero rate + dead = np.where(sparsity > 0.9)[0] + print(f" 'dead' dims (>90% near-zero): {len(dead)}/{n_dim} " + f"{('idx: ' + str(dead.tolist())) if 0 < len(dead) <= 20 else ''}") + weak = np.where(per_dim_var < 1e-4)[0] + print(f" 'weak' dims (var < 1e-4): {len(weak)}/{n_dim} " + f"{('idx: ' + str(weak.tolist())) if 0 < len(weak) <= 20 else ''}") + + +if __name__ == "__main__": + main() diff --git a/v4/scripts/analysis/summarize_run.py b/v4/scripts/analysis/summarize_run.py new file mode 100644 index 0000000..5b46c25 --- /dev/null +++ b/v4/scripts/analysis/summarize_run.py @@ -0,0 +1,132 @@ +"""summarize_run — print cross-rep stats for one v4 run folder. + +A "run" is a folder like ``v4/results/experiments/tri_v1/grid/bcd75_cw1_nt25/`` +containing ``rep00/``, ``rep01/``, ... — each with a per-rep ``summary.json`` +under any ``out_dir_tags`` subdir (typically ``binary/`` or ``binary/ntower/``). + +Usage: + python -m v4.scripts.analysis.summarize_run [--per-rep] [--json] + +Examples: + python -m v4.scripts.analysis.summarize_run \ + v4/results/experiments/tri_v1/grid/bcd75_cw1_nt25 + python -m v4.scripts.analysis.summarize_run \ + v4/results/experiments/tri_v1/baseline_tri --per-rep +""" +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + +import numpy as np + + +def find_rep_summaries(run_dir: Path) -> list[tuple[int, Path]]: + """Return [(rep_idx, summary_path), ...] sorted by rep_idx.""" + out: list[tuple[int, Path]] = [] + for rep_dir in sorted(run_dir.glob("rep*")): + if not rep_dir.is_dir(): + continue + m = re.match(r"rep(\d+)$", rep_dir.name) + if not m: + continue + summary = next(iter(rep_dir.rglob("summary.json")), None) + if summary is not None: + out.append((int(m.group(1)), summary)) + return out + + +def load_rep(summary_path: Path) -> dict: + """Extract the fields we summarise from one rep's summary.json.""" + d = json.loads(summary_path.read_text()) + return { + "val_mean": float(d.get("mean_val_auc", float("nan"))), + "val_std": float(d.get("std_val_auc", float("nan"))), + "test_mean": float(d.get("mean_test_auc", float("nan"))), + "test_std": float(d.get("std_test_auc", float("nan"))), + "elapsed_s": float(d.get("elapsed_s", float("nan"))), + "eval_stage": d.get("eval_stage", "?"), + } + + +def summarise(run_dir: Path) -> dict: + reps = find_rep_summaries(run_dir) + if not reps: + return {"run": str(run_dir), "n_reps": 0, "reps": []} + rows = [(idx, load_rep(p)) for idx, p in reps] + val = np.array([r[1]["val_mean"] for r in rows]) + test = np.array([r[1]["test_mean"] for r in rows]) + elaps = np.array([r[1]["elapsed_s"] for r in rows]) + out = { + "run": str(run_dir), + "n_reps": len(rows), + "eval_stage": rows[0][1]["eval_stage"], + "val_mean": float(np.mean(val)), + "val_std": float(np.std(val)), + "val_min": float(np.min(val)), + "val_max": float(np.max(val)), + "test_mean": float(np.mean(test)), + "test_std": float(np.std(test)), + "test_min": float(np.min(test)), + "test_max": float(np.max(test)), + "elapsed_total_s": float(np.sum(elaps)) if not np.isnan(elaps).any() else None, + "reps": [ + {"rep": idx, **info} for idx, info in rows + ], + } + return out + + +def render(s: dict, per_rep: bool = False) -> str: + if s["n_reps"] == 0: + return f"Run: {s['run']}\n no reps with summary.json found." + + lines = [ + f"Run: {s['run']}", + f"Reps: {s['n_reps']} (eval_stage={s['eval_stage']})", + f"Val AUC: {s['val_mean']:.4f} ± {s['val_std']:.4f} " + f"[min={s['val_min']:.4f} max={s['val_max']:.4f}]", + f"Test AUC: {s['test_mean']:.4f} ± {s['test_std']:.4f} " + f"[min={s['test_min']:.4f} max={s['test_max']:.4f}]", + ] + if s.get("elapsed_total_s") is not None: + h = s["elapsed_total_s"] / 3600 + lines.append(f"Compute: {s['elapsed_total_s']:.0f} s total ({h:.1f} h)") + + if per_rep: + lines.append("") + lines.append("Per-rep breakdown:") + lines.append(f" {'rep':>4s} {'val':>7s} {'test':>7s} {'elapsed':>7s}") + for r in s["reps"]: + elapsed = (f"{r['elapsed_s']:.0f}s" if not np.isnan(r['elapsed_s']) else "-") + lines.append( + f" {r['rep']:>4d} " + f"{r['val_mean']:>7.4f} " + f"{r['test_mean']:>7.4f} " + f"{elapsed:>7s}" + ) + return "\n".join(lines) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("run_dir", type=Path, help="Path to a run folder (contains repNN/ subdirs)") + ap.add_argument("--per-rep", action="store_true", help="Print one line per rep") + ap.add_argument("--json", action="store_true", help="Emit JSON instead of formatted text") + args = ap.parse_args() + + if not args.run_dir.is_dir(): + raise SystemExit(f"Not a directory: {args.run_dir}") + + s = summarise(args.run_dir) + if args.json: + print(json.dumps(s, indent=2)) + else: + print(render(s, per_rep=args.per_rep)) + + +if __name__ == "__main__": + main() diff --git a/v4/scripts/experiments/tri_v1/baseline_tri_features.json b/v4/scripts/experiments/tri_v1/baseline_tri_features.json new file mode 100644 index 0000000..34caa9a --- /dev/null +++ b/v4/scripts/experiments/tri_v1/baseline_tri_features.json @@ -0,0 +1,10 @@ +[ + { + "_note": "Single rep of tritower default with save_features=true so we can inspect nt-stage embeddings (Hadamard-product collapse hypothesis).", + "run_name": "experiments/tri_v1/baseline_tri_features", + "reps": 1, + "overrides": { + "save_features": true + } + } +] diff --git a/v4/scripts/experiments/tri_v1/cd_solo_geom_warm_sweep.json b/v4/scripts/experiments/tri_v1/cd_solo_geom_warm_sweep.json new file mode 100644 index 0000000..4ff0817 --- /dev/null +++ b/v4/scripts/experiments/tri_v1/cd_solo_geom_warm_sweep.json @@ -0,0 +1,13 @@ +[ + { "_note": "cd-solo + UNet geom-vector inject; cd_warm sweep. 3 reps each.", + "run_name": "experiments/tri_v1/cd_solo_geom/warm00", "reps": 3, + "stage_overrides": { "cd_warm": { "epochs": 0 } } }, + { "run_name": "experiments/tri_v1/cd_solo_geom/warm05", "reps": 3, + "stage_overrides": { "cd_warm": { "epochs": 5 } } }, + { "run_name": "experiments/tri_v1/cd_solo_geom/warm10", "reps": 3, + "stage_overrides": { "cd_warm": { "epochs": 10 } } }, + { "run_name": "experiments/tri_v1/cd_solo_geom/warm20", "reps": 3, + "stage_overrides": { "cd_warm": { "epochs": 20 } } }, + { "run_name": "experiments/tri_v1/cd_solo_geom/warm40", "reps": 3, + "stage_overrides": { "cd_warm": { "epochs": 40 } } } +] diff --git a/v4/scripts/experiments/tri_v1/cd_solo_warm_sweep.json b/v4/scripts/experiments/tri_v1/cd_solo_warm_sweep.json new file mode 100644 index 0000000..a3f8166 --- /dev/null +++ b/v4/scripts/experiments/tri_v1/cd_solo_warm_sweep.json @@ -0,0 +1,13 @@ +[ + { "_note": "cd-solo, no geom; cd_warm sweep. 3 reps each. Companion to cd_solo_geom sweep.", + "run_name": "experiments/tri_v1/cd_solo/warm00", "reps": 3, + "stage_overrides": { "cd_warm": { "epochs": 0 } } }, + { "run_name": "experiments/tri_v1/cd_solo/warm05", "reps": 3, + "stage_overrides": { "cd_warm": { "epochs": 5 } } }, + { "run_name": "experiments/tri_v1/cd_solo/warm10", "reps": 3, + "stage_overrides": { "cd_warm": { "epochs": 10 } } }, + { "run_name": "experiments/tri_v1/cd_solo/warm20", "reps": 3, + "stage_overrides": { "cd_warm": { "epochs": 20 } } }, + { "run_name": "experiments/tri_v1/cd_solo/warm40", "reps": 3, + "stage_overrides": { "cd_warm": { "epochs": 40 } } } +] diff --git a/v4/scripts/experiments/tri_v1/promote_bcd75_cw1_nt25.json b/v4/scripts/experiments/tri_v1/promote_bcd75_cw1_nt25.json new file mode 100644 index 0000000..1f9f1f8 --- /dev/null +++ b/v4/scripts/experiments/tri_v1/promote_bcd75_cw1_nt25.json @@ -0,0 +1,16 @@ +[ + { + "_note": "Promote grid winner bcd75_cw1_nt25 (3-rep test AUC 0.910) to 10 reps. First 3 reps will be skipped (already on disk).", + "run_name": "experiments/tri_v1/grid/bcd75_cw1_nt25", + "reps": 10, + "overrides": { + "training": { + "bcd_prob": 0.75, + "class_weighted": true + } + }, + "stage_overrides": { + "nt": { "epochs": 25 } + } + } +]