moved_repo_first_update
This commit is contained in:
rpotter6298
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Inspect saved validation/holdout logits for a multifold run.
|
||||
Prints per-class AUCs and sample counts so we can sanity-check unusually high scores.
|
||||
Can also print per-fold confusion matrices.
|
||||
|
||||
Example:
|
||||
python scripts/fold_confusion_matrix.py \
|
||||
--run-dir analysis_data/1030_Balanced_Unet_Perimg_Resnet_SE16NormB_SE16NormT_multi_fused/1030_Balanced_Unet_Perimg_Resnet_SE16NormB_SE16NormT_multi_fused_20251030_091842 \
|
||||
--head fused
|
||||
python scripts/fold_confusion_matrix.py --run-dir ... --head fused --use-holdout --confusion
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
import numpy as np
|
||||
from sklearn.metrics import roc_auc_score, confusion_matrix
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
ap = argparse.ArgumentParser(description="Inspect saved logits for a run and report per-class AUCs.")
|
||||
ap.add_argument("--run-dir", required=True, type=Path, help="Path to the run directory under analysis_data.")
|
||||
ap.add_argument("--head", choices=["fused", "image", "metadata"], default="fused",
|
||||
help="Which prediction head's saved probabilities to load.")
|
||||
ap.add_argument("--use-holdout", action="store_true",
|
||||
help="Look for *_holdout.npy dumps instead of validation splits.")
|
||||
ap.add_argument("--class-names", nargs="*", default=None,
|
||||
help="Optional override for class labels (order should match numeric labels).")
|
||||
ap.add_argument("--macro", action="store_true", help="Also print macro-average AUC across classes.")
|
||||
ap.add_argument("--confusion", action="store_true", help="Print confusion matrix for each fold.")
|
||||
return ap.parse_args()
|
||||
|
||||
|
||||
def load_cli_args(run_dir: Path) -> Dict:
|
||||
path = run_dir / "cli_args.json"
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Missing cli_args.json in {run_dir}")
|
||||
with path.open("r", encoding="utf-8") as fh:
|
||||
return json.load(fh)
|
||||
|
||||
|
||||
def find_fold_files(run_dir: Path, suffix: str) -> Dict[int, Dict[str, Path]]:
|
||||
files: Dict[int, Dict[str, Path]] = {}
|
||||
for y_file in run_dir.glob(f"fold*_y_true{suffix}.npy"):
|
||||
fold_str = y_file.stem.split("_")[0].replace("fold", "")
|
||||
try:
|
||||
fold_idx = int(fold_str)
|
||||
except ValueError:
|
||||
continue
|
||||
files.setdefault(fold_idx, {})["y_true"] = y_file
|
||||
for head_key, glob_pat in [
|
||||
("fused", f"fold*_probs_fused{suffix}.npy"),
|
||||
("image", f"fold*_probs_img{suffix}.npy"),
|
||||
("metadata", f"fold*_probs_md{suffix}.npy"),
|
||||
]:
|
||||
for p_file in run_dir.glob(glob_pat):
|
||||
fold_str = p_file.stem.split("_")[0].replace("fold", "")
|
||||
try:
|
||||
fold_idx = int(fold_str)
|
||||
except ValueError:
|
||||
continue
|
||||
files.setdefault(fold_idx, {})[head_key] = p_file
|
||||
return files
|
||||
|
||||
|
||||
def compute_auc(y_true: np.ndarray, probs: np.ndarray, class_names: List[str], macro: bool) -> List[int]:
|
||||
num_classes = probs.shape[1]
|
||||
unique = np.unique(y_true)
|
||||
print(f" classes present: {sorted(unique.tolist())}")
|
||||
|
||||
aucs = []
|
||||
seen_classes: List[int] = []
|
||||
for cls in range(num_classes):
|
||||
name = class_names[cls] if cls < len(class_names) else f"class_{cls}"
|
||||
mask = (y_true == cls)
|
||||
pos = int(mask.sum())
|
||||
neg = len(y_true) - pos
|
||||
if pos == 0 or neg == 0:
|
||||
print(f" {name:<15} -> insufficient positives/negatives (pos={pos}, neg={neg}); skipping AUC")
|
||||
continue
|
||||
try:
|
||||
auc = roc_auc_score((y_true == cls).astype(int), probs[:, cls])
|
||||
except ValueError as exc:
|
||||
print(f" {name:<15} -> AUC error: {exc}")
|
||||
continue
|
||||
aucs.append(auc)
|
||||
seen_classes.append(cls)
|
||||
print(f" {name:<15} -> AUC={auc:.4f} (pos={pos}, neg={neg})")
|
||||
|
||||
if macro and aucs:
|
||||
mean = float(np.mean(aucs))
|
||||
std = float(np.std(aucs, ddof=0)) if len(aucs) > 1 else math.nan
|
||||
print(f" macro AUC across reported classes: {mean:.4f} (std={std:.4f})")
|
||||
return seen_classes
|
||||
|
||||
|
||||
def print_confusion(y_true: np.ndarray, probs: np.ndarray, class_names: List[str]) -> None:
|
||||
num_classes = probs.shape[1]
|
||||
preds = probs.argmax(axis=1)
|
||||
labels = list(range(num_classes))
|
||||
cm = confusion_matrix(y_true, preds, labels=labels)
|
||||
names = [class_names[i] if i < len(class_names) else f"class_{i}" for i in labels]
|
||||
header = " " * 14 + "".join(f"{name:>12}" for name in names)
|
||||
print(" Confusion matrix (rows=true, cols=pred):")
|
||||
print(header)
|
||||
for idx, row in enumerate(cm):
|
||||
label = names[idx]
|
||||
row_str = "".join(f"{int(val):>12}" for val in row)
|
||||
print(f" {label:<12}{row_str}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
run_dir = args.run_dir.resolve()
|
||||
if not run_dir.exists():
|
||||
raise FileNotFoundError(run_dir)
|
||||
|
||||
cli_args = load_cli_args(run_dir)
|
||||
eval_mode = cli_args.get("eval_mode", "multiclass")
|
||||
if args.class_names:
|
||||
class_names = args.class_names
|
||||
else:
|
||||
if eval_mode == "binary":
|
||||
class_names = ["Healthy", "Glaucoma"]
|
||||
else:
|
||||
class_names = cli_args.get("class_names") or ["Healthy", "Glaucoma", "Suspect"]
|
||||
|
||||
suffix = "_holdout" if args.use_holdout else ""
|
||||
files = find_fold_files(run_dir, suffix)
|
||||
if not files:
|
||||
raise SystemExit(f"No saved probability files matching suffix '{suffix}' found in {run_dir}. "
|
||||
"Run scripts/rebuild_run_best_plots.py first if needed.")
|
||||
|
||||
print(f"[info] Inspecting head='{args.head}' ({'holdout' if args.use_holdout else 'validation'})")
|
||||
for fold_idx in sorted(files.keys()):
|
||||
fold = files[fold_idx]
|
||||
if "y_true" not in fold:
|
||||
print(f"[warning] Fold {fold_idx}: missing y_true file; skipping.")
|
||||
continue
|
||||
head_key = {
|
||||
"fused": "fused",
|
||||
"image": "image",
|
||||
"metadata": "metadata",
|
||||
}[args.head]
|
||||
prob_path = fold.get(head_key)
|
||||
if prob_path is None:
|
||||
print(f"[warning] Fold {fold_idx}: missing probability file for head '{args.head}'; skipping.")
|
||||
continue
|
||||
|
||||
y_true = np.load(fold["y_true"])
|
||||
probs = np.load(prob_path)
|
||||
print(f"\n Fold {fold_idx} -> samples={len(y_true)} file={prob_path.name}")
|
||||
compute_auc(y_true, probs, class_names, args.macro)
|
||||
if args.confusion:
|
||||
print_confusion(y_true, probs, class_names)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+356
@@ -0,0 +1,356 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Scan an analysis directory for HyperTower run folders, extract the best per-fold
|
||||
metric/accuracy from the epoch logs, and emit a combined summary.
|
||||
|
||||
Example:
|
||||
python scripts/batch_best_metrics.py \
|
||||
--analysis-dir analysis_data
|
||||
|
||||
# Holdout ranking (faster, uses summary.json):
|
||||
python scripts/batch_best_metrics.py \
|
||||
--analysis-dir analysis_data/grid_search \
|
||||
--metric holdout_auc_fused \
|
||||
--acc-metric holdout_acc_fused \
|
||||
--source summary \
|
||||
--sort-by mean_auc --desc --top 10
|
||||
|
||||
The script assumes each run directory contains files named `fold{n}_epoch_log.csv`.
|
||||
It reports runs that have all five folds (fold0..fold4) present by default.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
REQUIRED_FOLDS = {f"fold{i}_epoch_log.csv" for i in range(5)}
|
||||
|
||||
|
||||
def to_float(value: Optional[object]) -> Optional[float]:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
num = float(value)
|
||||
if math.isnan(num):
|
||||
return None
|
||||
return num
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
value = value.strip()
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
num = float(value)
|
||||
except ValueError:
|
||||
return None
|
||||
if math.isnan(num):
|
||||
return None
|
||||
return num
|
||||
|
||||
|
||||
def best_value_from_csv(csv_path: Path, metric: str) -> Optional[Tuple[float, int]]:
|
||||
best: Optional[Tuple[float, int]] = None
|
||||
with csv_path.open("r", newline="") as fp:
|
||||
reader = csv.DictReader(fp)
|
||||
for row in reader:
|
||||
val = to_float(row.get(metric))
|
||||
if val is None:
|
||||
continue
|
||||
epoch = int(to_float(row.get("epoch")) or reader.line_num)
|
||||
if best is None or val > best[0]:
|
||||
best = (val, epoch)
|
||||
return best
|
||||
|
||||
|
||||
def render_progress(current: int, total: Optional[int], matched: int) -> str:
|
||||
if total:
|
||||
width = 30
|
||||
filled = int(width * current / total)
|
||||
bar = "#" * filled + "-" * (width - filled)
|
||||
return f"[{bar}] {current}/{total} matched {matched}"
|
||||
return f"Scanned {current} dirs, matched {matched}"
|
||||
|
||||
|
||||
def find_run_directories(root: Path,
|
||||
shallow: bool,
|
||||
required_files: Iterable[str],
|
||||
show_progress: bool) -> Iterable[Path]:
|
||||
"""
|
||||
Yield directories that look like HyperTower runs (contain at least the required fold logs).
|
||||
"""
|
||||
required_set = set(required_files)
|
||||
if shallow:
|
||||
entries = [entry for entry in root.iterdir() if entry.is_dir()]
|
||||
entries.sort(key=lambda p: p.name)
|
||||
total = len(entries)
|
||||
matched = 0
|
||||
last_update = 0.0
|
||||
for idx, entry in enumerate(entries, start=1):
|
||||
if show_progress:
|
||||
now = time.monotonic()
|
||||
if now - last_update >= 0.1 or idx == total:
|
||||
msg = render_progress(idx, total, matched)
|
||||
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
|
||||
last_update = now
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
if all((entry / filename).is_file() for filename in required_set):
|
||||
matched += 1
|
||||
yield entry
|
||||
if show_progress:
|
||||
print(file=sys.stderr)
|
||||
return
|
||||
|
||||
matched = 0
|
||||
scanned = 0
|
||||
last_update = 0.0
|
||||
for dirpath, dirnames, filenames in os_walk_sorted(root):
|
||||
scanned += 1
|
||||
if show_progress:
|
||||
now = time.monotonic()
|
||||
if now - last_update >= 0.2:
|
||||
msg = render_progress(scanned, None, matched)
|
||||
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
|
||||
last_update = now
|
||||
files = set(filenames)
|
||||
if required_set.issubset(files):
|
||||
matched += 1
|
||||
yield Path(dirpath)
|
||||
if show_progress:
|
||||
msg = render_progress(scanned, None, matched)
|
||||
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
|
||||
print(file=sys.stderr)
|
||||
|
||||
|
||||
def os_walk_sorted(root: Path):
|
||||
"""
|
||||
Wrapper around os.walk that yields deterministic, sorted directory order.
|
||||
"""
|
||||
import os
|
||||
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
dirnames.sort()
|
||||
filenames.sort()
|
||||
yield dirpath, dirnames, filenames
|
||||
|
||||
|
||||
def read_summary(run_dir: Path) -> Optional[Dict[str, object]]:
|
||||
summary_path = run_dir / "summary.json"
|
||||
if not summary_path.exists():
|
||||
return None
|
||||
try:
|
||||
data = json.loads(summary_path.read_text())
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
def read_run_id(run_dir: Path, summary: Optional[Dict[str, object]] = None) -> str:
|
||||
data = summary if summary is not None else read_summary(run_dir)
|
||||
if data:
|
||||
rid = data.get("run_id")
|
||||
if isinstance(rid, str) and rid:
|
||||
return rid
|
||||
return run_dir.name
|
||||
|
||||
|
||||
def mean(values: List[float]) -> Optional[float]:
|
||||
return (sum(values) / len(values)) if values else None
|
||||
|
||||
|
||||
def metric_from_stats(stats: Dict[str, object], metric: str) -> Optional[float]:
|
||||
if stats.get("holdout_best_monitor") == metric:
|
||||
best_val = to_float(stats.get("holdout_best_so_far"))
|
||||
if best_val is not None:
|
||||
return best_val
|
||||
return to_float(stats.get(metric))
|
||||
|
||||
def task_from_summary(summary: Optional[Dict[str, object]]) -> Optional[str]:
|
||||
if not summary:
|
||||
return None
|
||||
eval_mode = summary.get("eval_mode")
|
||||
if isinstance(eval_mode, str):
|
||||
mode = eval_mode.strip().lower()
|
||||
if mode == "binary":
|
||||
return "binary"
|
||||
if mode in {"multiclass", "multi", "multi-class"}:
|
||||
return "multiclass"
|
||||
num_classes = summary.get("num_classes")
|
||||
if isinstance(num_classes, (int, float)):
|
||||
return "binary" if int(num_classes) <= 2 else "multiclass"
|
||||
return None
|
||||
|
||||
|
||||
def format_table(rows: List[Dict[str, Optional[object]]], columns: List[str]) -> str:
|
||||
col_widths = {
|
||||
col: max(len(col), max((len(fmt_value(row.get(col))) for row in rows), default=0))
|
||||
for col in columns
|
||||
}
|
||||
header = " | ".join(col.ljust(col_widths[col]) for col in columns)
|
||||
divider = "-+-".join("-" * col_widths[col] for col in columns)
|
||||
body_lines = [
|
||||
" | ".join(fmt_value(row.get(col)).ljust(col_widths[col]) for col in columns)
|
||||
for row in rows
|
||||
]
|
||||
return "\n".join([header, divider, *body_lines])
|
||||
|
||||
|
||||
def fmt_value(value: Optional[object]) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
if isinstance(value, int):
|
||||
return str(value)
|
||||
return f"{value:.4f}"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description="Aggregate best per-fold metrics from HyperTower runs.")
|
||||
ap.add_argument("--analysis-dir", type=Path, default=Path("analysis_data"),
|
||||
help="Directory containing run subdirectories (default: analysis_data)")
|
||||
ap.add_argument("--metric", default="auc_fused",
|
||||
help="Metric column to maximise (default: auc_fused)")
|
||||
ap.add_argument("--acc-metric", default="acc_fused",
|
||||
help="Accuracy column to maximise (default: acc_fused)")
|
||||
ap.add_argument("--shallow", action="store_true",
|
||||
help="Only scan directories directly under analysis-dir")
|
||||
ap.add_argument("--source", choices=["epoch_logs", "summary"], default="epoch_logs",
|
||||
help="Where to read metrics from (default: epoch_logs)")
|
||||
ap.add_argument("--task", choices=["binary", "multiclass", "all"], default="all",
|
||||
help="Filter runs by task type (default: all)")
|
||||
ap.add_argument("--no-progress", action="store_true",
|
||||
help="Disable progress output")
|
||||
ap.add_argument("--match", default=None,
|
||||
help="Only include run directories whose name contains this substring")
|
||||
ap.add_argument("--sort-by", choices=["mean_auc", "mean_acc"], default=None,
|
||||
help="Optional column to sort by (default: none)")
|
||||
ap.add_argument("--desc", action="store_true",
|
||||
help="Sort in descending order (default: ascending)")
|
||||
ap.add_argument("--top", type=int, default=None,
|
||||
help="Limit output to the top N rows after sorting")
|
||||
ap.add_argument("--output-file", type=Path, default=None,
|
||||
help="Optional path to write CSV summary")
|
||||
args = ap.parse_args()
|
||||
|
||||
root = args.analysis_dir
|
||||
if not root.exists():
|
||||
raise SystemExit(f"Analysis directory not found: {root}")
|
||||
|
||||
rows: List[Dict[str, Optional[object]]] = []
|
||||
missing_summary = 0
|
||||
unknown_task = 0
|
||||
|
||||
required_files = REQUIRED_FOLDS if args.source == "epoch_logs" else ["summary.json"]
|
||||
for run_dir in find_run_directories(
|
||||
root,
|
||||
shallow=args.shallow,
|
||||
required_files=required_files,
|
||||
show_progress=not args.no_progress,
|
||||
):
|
||||
if args.match and args.match not in run_dir.name:
|
||||
continue
|
||||
summary = None
|
||||
task_label = None
|
||||
if args.task != "all" or args.source == "summary":
|
||||
summary = read_summary(run_dir)
|
||||
if summary is None:
|
||||
missing_summary += 1
|
||||
continue
|
||||
task_label = task_from_summary(summary)
|
||||
if args.task != "all":
|
||||
if task_label is None:
|
||||
unknown_task += 1
|
||||
continue
|
||||
if task_label != args.task:
|
||||
continue
|
||||
|
||||
run_id = read_run_id(run_dir, summary)
|
||||
best_metrics: List[float] = []
|
||||
best_accs: List[float] = []
|
||||
if args.source == "summary":
|
||||
folds = summary.get("fold_metrics") if summary else None
|
||||
if not folds:
|
||||
continue
|
||||
for fold in folds:
|
||||
stats = fold.get("stats") or {}
|
||||
metric_val = metric_from_stats(stats, args.metric)
|
||||
acc_val = metric_from_stats(stats, args.acc_metric)
|
||||
if metric_val is None or acc_val is None:
|
||||
best_metrics = []
|
||||
best_accs = []
|
||||
break
|
||||
best_metrics.append(metric_val)
|
||||
best_accs.append(acc_val)
|
||||
else:
|
||||
for fold_idx in range(5):
|
||||
csv_path = run_dir / f"fold{fold_idx}_epoch_log.csv"
|
||||
metric_entry = best_value_from_csv(csv_path, args.metric)
|
||||
acc_entry = best_value_from_csv(csv_path, args.acc_metric)
|
||||
if metric_entry is None or acc_entry is None:
|
||||
# Skip this run if any fold is missing data
|
||||
best_metrics = []
|
||||
best_accs = []
|
||||
break
|
||||
best_metrics.append(metric_entry[0])
|
||||
best_accs.append(acc_entry[0])
|
||||
|
||||
if not best_metrics or not best_accs:
|
||||
continue
|
||||
|
||||
rows.append({
|
||||
"run_id": run_id,
|
||||
"task": task_label,
|
||||
"relative_path": str(run_dir.relative_to(root)),
|
||||
"mean_auc": mean(best_metrics),
|
||||
"mean_acc": mean(best_accs),
|
||||
})
|
||||
|
||||
if not rows:
|
||||
print("No matching runs found.")
|
||||
return
|
||||
|
||||
if args.sort_by:
|
||||
def sort_key(row: Dict[str, Optional[float]]) -> float:
|
||||
value = row.get(args.sort_by)
|
||||
if value is None:
|
||||
return float("-inf") if args.desc else float("inf")
|
||||
return float(value)
|
||||
|
||||
rows.sort(key=sort_key, reverse=args.desc)
|
||||
|
||||
if args.top is not None:
|
||||
rows = rows[:args.top]
|
||||
|
||||
columns = ["run_id", "task", "relative_path", "mean_auc", "mean_acc"]
|
||||
if args.task != "all":
|
||||
print(f"Task filter: {args.task}")
|
||||
if args.match:
|
||||
print(f"Name filter: {args.match}")
|
||||
print(f"Runs: {len(rows)}\n")
|
||||
print(format_table(rows, columns))
|
||||
|
||||
if args.output_file:
|
||||
out_path = args.output_file
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with out_path.open("w", newline="") as fp:
|
||||
writer = csv.DictWriter(fp, fieldnames=columns)
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
writer.writerow(row)
|
||||
print(f"\nSummary written to {out_path}")
|
||||
if args.task != "all" and (missing_summary or unknown_task):
|
||||
print(f"\nSkipped {missing_summary} runs without summary.json and {unknown_task} with unknown task type.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,286 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Aggregate per-fold metrics across runs and visualize AUC vs accuracy.
|
||||
|
||||
The script scans every `summary.json` under the provided analysis directory,
|
||||
loads the per-fold macro AUC values, and combines them with per-fold
|
||||
predictions to compute accuracy. Two scatter plots are produced:
|
||||
|
||||
1. AUC vs. fold index (with jitter) coloured by fold.
|
||||
2. Accuracy (x-axis) vs. AUC (y-axis) coloured by fold.
|
||||
|
||||
This helps identify folds that persistently underperform across experiments.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, List, Optional
|
||||
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.cm import get_cmap
|
||||
from matplotlib.lines import Line2D
|
||||
except ImportError as exc: # pragma: no cover - forward-friendly error for runtime
|
||||
raise SystemExit("matplotlib is required to run this script") from exc
|
||||
|
||||
|
||||
@dataclass
|
||||
class FoldMetric:
|
||||
run_id: str
|
||||
fold: int
|
||||
auc: float
|
||||
accuracy: float
|
||||
summary_path: Path
|
||||
fusion_mode: Optional[str]
|
||||
plot_head: str
|
||||
|
||||
|
||||
HEAD_SUFFIX = {
|
||||
"fused": "fused",
|
||||
"metadata": "md",
|
||||
"metadata_only": "md",
|
||||
"image": "img",
|
||||
"image_only": "img",
|
||||
"img": "img",
|
||||
"md": "md",
|
||||
}
|
||||
|
||||
|
||||
def infer_head(summary: Dict[str, object]) -> str:
|
||||
"""Return the prediction head name used for evaluation."""
|
||||
plot_head = summary.get("plot_head")
|
||||
if isinstance(plot_head, str) and plot_head:
|
||||
key = plot_head.lower()
|
||||
if key in HEAD_SUFFIX:
|
||||
return key
|
||||
fusion_mode = summary.get("fusion_mode")
|
||||
if isinstance(fusion_mode, str):
|
||||
key = fusion_mode.lower()
|
||||
if key in HEAD_SUFFIX:
|
||||
return key
|
||||
# Fall back to fused head if nothing else matches
|
||||
return "fused"
|
||||
|
||||
|
||||
def prediction_suffix(head: str) -> str:
|
||||
key = head.lower()
|
||||
if key in {"metadata", "metadata_only", "md"}:
|
||||
return "md"
|
||||
if key in {"image", "image_only", "img"}:
|
||||
return "img"
|
||||
return "fused"
|
||||
|
||||
|
||||
def compute_accuracy(probs: np.ndarray, y_true: np.ndarray) -> float:
|
||||
if probs.ndim == 1:
|
||||
preds = (probs >= 0.5).astype(int)
|
||||
else:
|
||||
preds = np.argmax(probs, axis=1)
|
||||
y_int = y_true.astype(int)
|
||||
return float((preds == y_int).mean()) if y_int.size else np.nan
|
||||
|
||||
|
||||
def load_summary(path: Path) -> Optional[Dict[str, object]]:
|
||||
try:
|
||||
with path.open("r") as f:
|
||||
return json.load(f)
|
||||
except Exception as exc:
|
||||
print(f"[warn] Could not parse {path}: {exc}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
def collect_metrics(summary_path: Path) -> Iterable[FoldMetric]:
|
||||
summary = load_summary(summary_path)
|
||||
if not summary:
|
||||
return []
|
||||
# Only keep multiclass experiments (num_classes > 2 or eval_mode explicitly multiclass)
|
||||
num_classes = summary.get("num_classes")
|
||||
eval_mode = summary.get("eval_mode")
|
||||
if (isinstance(num_classes, int) and num_classes <= 2) or (isinstance(eval_mode, str) and eval_mode.lower() == "binary"):
|
||||
return []
|
||||
|
||||
head = infer_head(summary)
|
||||
per_fold_auc = summary.get("per_fold_macro_ovr_auc") or summary.get("per_fold_auc")
|
||||
if not isinstance(per_fold_auc, list):
|
||||
# Fallback for summaries that only store fold_metrics[*].stats.
|
||||
metric_key = f"auc_{prediction_suffix(head)}"
|
||||
fold_metrics = summary.get("fold_metrics")
|
||||
if not isinstance(fold_metrics, list):
|
||||
return []
|
||||
per_fold_auc = []
|
||||
for entry in fold_metrics:
|
||||
if not isinstance(entry, dict):
|
||||
return []
|
||||
stats = entry.get("stats")
|
||||
if not isinstance(stats, dict):
|
||||
return []
|
||||
auc_val = stats.get(metric_key)
|
||||
try:
|
||||
per_fold_auc.append(float(auc_val))
|
||||
except (TypeError, ValueError):
|
||||
return []
|
||||
|
||||
suffix = prediction_suffix(head)
|
||||
run_id = summary.get("run_id", summary_path.parent.name)
|
||||
fusion_mode = summary.get("fusion_mode")
|
||||
|
||||
for fold_idx, auc_val in enumerate(per_fold_auc):
|
||||
try:
|
||||
auc = float(auc_val)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
|
||||
base = summary_path.parent
|
||||
probs_path = base / f"fold{fold_idx}_probs_{suffix}.npy"
|
||||
y_true_path = base / f"fold{fold_idx}_y_true.npy"
|
||||
if not probs_path.exists() or not y_true_path.exists():
|
||||
# fall back: if fused missing for metadata mode (or vice versa), try md or img
|
||||
if suffix != "fused":
|
||||
alt_probs_path = base / f"fold{fold_idx}_probs_fused.npy"
|
||||
if alt_probs_path.exists():
|
||||
probs_path = alt_probs_path
|
||||
if not probs_path.exists():
|
||||
print(
|
||||
f"[warn] Missing predictions for fold {fold_idx} in {base}; skipped",
|
||||
file=sys.stderr,
|
||||
)
|
||||
continue
|
||||
try:
|
||||
probs = np.load(probs_path)
|
||||
y_true = np.load(y_true_path)
|
||||
except Exception as exc:
|
||||
print(f"[warn] Failed loading predictions for {base}: {exc}", file=sys.stderr)
|
||||
continue
|
||||
accuracy = compute_accuracy(probs, y_true)
|
||||
yield FoldMetric(
|
||||
run_id=str(run_id),
|
||||
fold=fold_idx,
|
||||
auc=auc,
|
||||
accuracy=accuracy,
|
||||
summary_path=summary_path,
|
||||
fusion_mode=fusion_mode if isinstance(fusion_mode, str) else None,
|
||||
plot_head=head,
|
||||
)
|
||||
|
||||
|
||||
def build_plot(metrics: List[FoldMetric], output: Path, jitter: float, seed: int, show: bool) -> None:
|
||||
rng = np.random.default_rng(seed)
|
||||
folds = sorted({m.fold for m in metrics})
|
||||
fold_to_color: Dict[int, tuple] = {}
|
||||
cmap = get_cmap("tab10", max(len(folds), 1))
|
||||
for idx, fold in enumerate(folds):
|
||||
fold_to_color[fold] = cmap(idx)
|
||||
|
||||
# Prepare arrays for plotting
|
||||
aucs = np.array([m.auc for m in metrics])
|
||||
accs = np.array([m.accuracy for m in metrics])
|
||||
fold_indices = np.array([m.fold for m in metrics])
|
||||
colors = [fold_to_color[m.fold] for m in metrics]
|
||||
jitter_offsets = rng.uniform(-jitter, jitter, size=len(metrics))
|
||||
|
||||
fig, axes = plt.subplots(1, 2, figsize=(13, 5), constrained_layout=True)
|
||||
|
||||
# Panel 1: Fold vs AUC scatter with jitter
|
||||
ax0 = axes[0]
|
||||
ax0.scatter(fold_indices + 1 + jitter_offsets, aucs, c=colors, edgecolor="k", linewidth=0.4, alpha=0.85)
|
||||
ax0.set_xticks([f + 1 for f in folds])
|
||||
ax0.set_xlabel("Fold index")
|
||||
ax0.set_ylabel("Macro AUC")
|
||||
ax0.set_title("Per-fold AUC across runs")
|
||||
ax0.grid(True, linestyle=":", linewidth=0.5, alpha=0.4)
|
||||
|
||||
# Panel 2: Accuracy vs AUC scatter
|
||||
ax1 = axes[1]
|
||||
ax1.scatter(accs, aucs, c=colors, edgecolor="k", linewidth=0.4, alpha=0.85)
|
||||
ax1.set_xlabel("Accuracy")
|
||||
ax1.set_ylabel("Macro AUC")
|
||||
ax1.set_title("Accuracy vs AUC by fold")
|
||||
ax1.grid(True, linestyle=":", linewidth=0.5, alpha=0.4)
|
||||
|
||||
# Shared legend
|
||||
legend_handles = [
|
||||
Line2D(
|
||||
[0],
|
||||
[0],
|
||||
marker="o",
|
||||
color="w",
|
||||
label=f"Fold {fold + 1}",
|
||||
markerfacecolor=fold_to_color[fold],
|
||||
markeredgecolor="k",
|
||||
markersize=8,
|
||||
)
|
||||
for fold in folds
|
||||
]
|
||||
for ax in axes:
|
||||
ax.legend(handles=legend_handles, frameon=False, loc="lower right")
|
||||
|
||||
fig.suptitle("Fold-level performance across experiments", fontsize=14)
|
||||
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(output, dpi=200)
|
||||
print(f"Saved plot to {output}")
|
||||
|
||||
if show:
|
||||
plt.show()
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def print_summary(metrics: List[FoldMetric]) -> None:
|
||||
total_runs = len({m.run_id for m in metrics})
|
||||
print(f"Collected {len(metrics)} fold metrics from {total_runs} runs.")
|
||||
by_fold: Dict[int, List[FoldMetric]] = {}
|
||||
for metric in metrics:
|
||||
by_fold.setdefault(metric.fold, []).append(metric)
|
||||
for fold, entries in sorted(by_fold.items()):
|
||||
aucs = np.array([m.auc for m in entries])
|
||||
accs = np.array([m.accuracy for m in entries])
|
||||
print(
|
||||
f" Fold {fold + 1}: AUC {aucs.mean():.3f} ± {aucs.std(ddof=0):.3f} | "
|
||||
f"Accuracy {accs.mean():.3f} ± {accs.std(ddof=0):.3f} (n={len(entries)})"
|
||||
)
|
||||
|
||||
|
||||
def main(argv: Optional[List[str]] = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Plot per-fold AUCs and accuracies across runs.")
|
||||
parser.add_argument(
|
||||
"--analysis-root",
|
||||
default="analysis_data",
|
||||
help="Root directory that contains run folders with summary.json files (default: analysis_data)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default="analysis_data/fold_auc_vs_accuracy.png",
|
||||
help="Where to save the generated figure (default: analysis_data/fold_auc_vs_accuracy.png)",
|
||||
)
|
||||
parser.add_argument("--jitter", type=float, default=0.08, help="Horizontal jitter for fold scatter plot")
|
||||
parser.add_argument("--seed", type=int, default=17, help="Random seed for jitter replication")
|
||||
parser.add_argument("--show", action="store_true", help="Display the plot interactively after saving")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
analysis_root = Path(args.analysis_root)
|
||||
if not analysis_root.exists():
|
||||
raise SystemExit(f"Analysis root {analysis_root} does not exist")
|
||||
|
||||
summary_files = sorted(analysis_root.rglob("summary.json"))
|
||||
if not summary_files:
|
||||
raise SystemExit(f"No summary.json files found under {analysis_root}")
|
||||
|
||||
metrics: List[FoldMetric] = []
|
||||
for summary_path in summary_files:
|
||||
metrics.extend(collect_metrics(summary_path))
|
||||
|
||||
if not metrics:
|
||||
raise SystemExit("No fold metrics collected. Check that prediction files are present.")
|
||||
|
||||
print_summary(metrics)
|
||||
build_plot(metrics, Path(args.output), jitter=args.jitter, seed=args.seed, show=args.show)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
"""Filter segmentation metrics rows with near-zero Dice scores."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Drop samples where both disc and cup Dice are below a threshold "
|
||||
"(default 0.01) and report how many were removed."
|
||||
)
|
||||
)
|
||||
parser.add_argument("input", type=Path, help="Path to metrics CSV to filter")
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
help="Destination CSV. Defaults to <input stem>_filtered.csv in the same directory.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--threshold",
|
||||
type=float,
|
||||
default=0.01,
|
||||
help="Dice cutoff; rows with both dice_disc and dice_cup below this are removed.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--keep-summary",
|
||||
action="store_true",
|
||||
help="Always keep summary rows (sample_id == '__mean__').",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
df = pd.read_csv(args.input)
|
||||
|
||||
mask_low = (df["dice_disc"] < args.threshold) & (df["dice_cup"] < args.threshold)
|
||||
if args.keep_summary and "sample_id" in df.columns:
|
||||
mask_low &= df["sample_id"].ne("__mean__")
|
||||
|
||||
removed = int(mask_low.sum())
|
||||
filtered = df.loc[~mask_low].copy()
|
||||
|
||||
# Recompute summary if original file contained one
|
||||
if "sample_id" in filtered.columns:
|
||||
summary_mask = filtered["sample_id"].eq("__mean__")
|
||||
filtered = filtered.loc[~summary_mask].copy()
|
||||
if not filtered.empty:
|
||||
summary = filtered[["dice_disc", "dice_cup"]].mean()
|
||||
summary_row = {
|
||||
"sample_id": "__mean__",
|
||||
"dataset": "summary",
|
||||
"split": "summary",
|
||||
"dice_disc": summary["dice_disc"],
|
||||
"dice_cup": summary["dice_cup"],
|
||||
}
|
||||
filtered = pd.concat([filtered, pd.DataFrame([summary_row])], ignore_index=True)
|
||||
|
||||
remaining = len(filtered)
|
||||
|
||||
output_path = args.output
|
||||
if output_path is None:
|
||||
output_path = args.input.with_name(f"{args.input.stem}_filtered.csv")
|
||||
|
||||
filtered.to_csv(output_path, index=False)
|
||||
|
||||
print(f"Removed rows: {removed}")
|
||||
print(f"Remaining rows: {remaining}")
|
||||
print(f"Filtered metrics saved to: {output_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+241
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Per-class ROC: one figure per class (multiclass) OR one figure total (binary),
|
||||
with ALL models (runs under a tag) plotted as separate lines.
|
||||
|
||||
Outputs under analysis_data/:
|
||||
- multiclass:
|
||||
<tag>_class0_roc.png (e.g., Healthy)
|
||||
<tag>_class1_roc.png (e.g., Glaucoma)
|
||||
<tag>_class2_roc.png (e.g., Suspect)
|
||||
<tag>_perclass_summary.json
|
||||
- binary:
|
||||
<tag>_binary_roc.png
|
||||
<tag>_perclass_summary.json
|
||||
"""
|
||||
|
||||
import argparse, json, re
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.metrics import roc_curve, auc, roc_auc_score
|
||||
|
||||
HEAD_ALIASES = {"image": ["image","img"], "fused": ["fused"], "metadata": ["metadata","md"]}
|
||||
|
||||
def find_run_dirs(tag_prefix: str, analysis_dir: Path):
|
||||
return sorted([p for p in analysis_dir.glob(f"{tag_prefix}_*") if p.is_dir()])
|
||||
|
||||
def read_summary(run_dir: Path) -> dict:
|
||||
p = run_dir / "summary.json"
|
||||
if p.exists():
|
||||
try:
|
||||
return json.loads(p.read_text())
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
def find_folds(run_dir: Path, head: str):
|
||||
variants = HEAD_ALIASES.get(head, [head])
|
||||
y_files = sorted(run_dir.glob("fold*_y_true.npy"))
|
||||
folds = []
|
||||
for yf in y_files:
|
||||
m = re.search(r"fold(\d+)_y_true\.npy$", yf.name)
|
||||
if not m: continue
|
||||
idx = int(m.group(1))
|
||||
if any((run_dir / f"fold{idx}_probs_{v}.npy").exists() for v in variants):
|
||||
folds.append(idx)
|
||||
return folds
|
||||
|
||||
def load_probs(run_dir: Path, fold: int, head: str):
|
||||
variants = HEAD_ALIASES.get(head, [head])
|
||||
y = np.load(run_dir / f"fold{fold}_y_true.npy")
|
||||
p = None
|
||||
tried = []
|
||||
for v in variants:
|
||||
pp = run_dir / f"fold{fold}_probs_{v}.npy"
|
||||
tried.append(pp.name)
|
||||
if pp.exists():
|
||||
p = np.load(pp); break
|
||||
if p is None:
|
||||
raise FileNotFoundError(f"Missing probs for fold {fold} in {run_dir}; tried {tried}")
|
||||
return y, p
|
||||
|
||||
def infer_mode_from_files(run_dir: Path, head: str):
|
||||
f = find_folds(run_dir, head)
|
||||
if not f: return None
|
||||
_, p = load_probs(run_dir, f[0], head)
|
||||
if p.ndim == 2 and p.shape[1] == 2: return "binary"
|
||||
if p.ndim == 2 and p.shape[1] >= 3: return "multiclass"
|
||||
return None
|
||||
|
||||
def per_class_roc(y, p):
|
||||
"""Return {k: (fpr, tpr, auc)} for OVR."""
|
||||
K = p.shape[1]
|
||||
out = {}
|
||||
for k in range(K):
|
||||
yb = (y == k).astype(np.uint8)
|
||||
fpr, tpr, _ = roc_curve(yb, p[:, k])
|
||||
out[k] = (fpr, tpr, auc(fpr, tpr) if len(fpr) > 1 else np.nan)
|
||||
return out
|
||||
|
||||
def make_per_model_class_curves(run_dir: Path, head: str, mode: str):
|
||||
"""
|
||||
Returns:
|
||||
label (model/backbone name),
|
||||
class_curves: dict[k] -> dict with keys:
|
||||
'fpr': grid, 'tpr_mean': mean across folds on grid, 'auc_mean': mean across folds,
|
||||
'tpr_std' and 'auc_std' also included.
|
||||
K = number of classes (2 or 3+)
|
||||
"""
|
||||
summary = read_summary(run_dir)
|
||||
label = summary.get("backbone") or run_dir.name
|
||||
folds = find_folds(run_dir, head)
|
||||
if not folds:
|
||||
return None
|
||||
|
||||
# collect per-fold per-class curves
|
||||
per_fold = []
|
||||
for f in folds:
|
||||
y, p = load_probs(run_dir, f, head)
|
||||
if mode == "binary":
|
||||
keep = np.isin(y, [0,1])
|
||||
if keep.sum() == 0:
|
||||
continue
|
||||
y, p = y[keep], p[keep]
|
||||
if p.shape[1] > 2: # safety; binary should have 2 cols
|
||||
p = p[:, :2]
|
||||
else:
|
||||
if p.ndim != 2 or p.shape[1] < 3:
|
||||
continue
|
||||
per_fold.append(per_class_roc(y, p))
|
||||
if not per_fold:
|
||||
return None
|
||||
|
||||
# interpolate on a common grid, avg across folds
|
||||
grid = np.linspace(0, 1, 501)
|
||||
K = max(per_fold[0].keys()) + 1
|
||||
class_curves = {}
|
||||
for k in range(K):
|
||||
tprs, aucs = [], []
|
||||
for d in per_fold:
|
||||
if k not in d:
|
||||
continue
|
||||
fpr, tpr, a = d[k]
|
||||
tprs.append(np.interp(grid, fpr, tpr))
|
||||
aucs.append(a)
|
||||
if not tprs:
|
||||
continue
|
||||
tprs = np.vstack(tprs)
|
||||
class_curves[k] = {
|
||||
"fpr": grid,
|
||||
"tpr_mean": tprs.mean(axis=0),
|
||||
"tpr_std": tprs.std(axis=0),
|
||||
"auc_mean": float(np.nanmean(aucs)),
|
||||
"auc_std": float(np.nanstd(aucs)),
|
||||
}
|
||||
return label, class_curves
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Per-class ROC with all models as separate lines.")
|
||||
ap.add_argument("--tag", required=True, help="analysis_data prefix like 'papergrid'")
|
||||
ap.add_argument("--head", default="image", choices=["image","fused","metadata"])
|
||||
ap.add_argument("--mode", choices=["binary","multiclass"], required=True,
|
||||
help="Select which experiment style to aggregate.")
|
||||
ap.add_argument("--fusion-mode", choices=["image_only","fused","metadata_only","vote"], default=None,
|
||||
help="Filter runs by fusion mode to avoid mixing.")
|
||||
ap.add_argument("--analysis-dir", default="analysis_data")
|
||||
ap.add_argument("--class-names", nargs="*", default=["Healthy","Glaucoma","Suspect"])
|
||||
ap.add_argument("--shade", action="store_true", help="Shade ±1 SD per model (can get busy).")
|
||||
args = ap.parse_args()
|
||||
|
||||
analysis_dir = Path(args.analysis_dir) / args.tag
|
||||
run_dirs_all = find_run_dirs(args.tag, analysis_dir)
|
||||
if not run_dirs_all:
|
||||
raise SystemExit(f"No run directories found starting with '{args.tag}_' under {analysis_dir}")
|
||||
|
||||
# filter runs
|
||||
selected = []
|
||||
skipped = []
|
||||
for rd in run_dirs_all:
|
||||
sj = read_summary(rd)
|
||||
m = sj.get("eval_mode") or infer_mode_from_files(rd, args.head)
|
||||
if m != args.mode:
|
||||
skipped.append((rd, f"mode={m}")); continue
|
||||
if args.fusion_mode:
|
||||
fm = sj.get("fusion_mode")
|
||||
if fm and fm != args.fusion_mode:
|
||||
skipped.append((rd, f"fusion_mode={fm}")); continue
|
||||
selected.append(rd)
|
||||
|
||||
if not selected:
|
||||
raise SystemExit("No runs matched filters (mode/fusion-mode).")
|
||||
|
||||
# build per-model curves
|
||||
per_model = [] # list of (label, class_curves)
|
||||
for rd in selected:
|
||||
res = make_per_model_class_curves(rd, args.head, args.mode)
|
||||
if res is None:
|
||||
skipped.append((rd, "no_usable_folds")); continue
|
||||
per_model.append(res)
|
||||
|
||||
if not per_model:
|
||||
raise SystemExit("No usable runs after fold parsing/interpolation.")
|
||||
|
||||
# determine classes to plot
|
||||
maxK = max((max(curves.keys())+1) for _, curves in per_model)
|
||||
if args.mode == "binary":
|
||||
# Only class 1 (positive) is typically plotted
|
||||
classes_to_plot = [1]
|
||||
class_names = [args.class_names[1] if len(args.class_names) > 1 else "Positive"]
|
||||
outfile_names = [f"{args.tag}_binary_roc.png"]
|
||||
title_suffixes = ["Binary (positive class)"]
|
||||
else:
|
||||
classes_to_plot = list(range(min(3, maxK))) # usually 0,1,2
|
||||
class_names = [args.class_names[i] if i < len(args.class_names) else f"class {i}" for i in classes_to_plot]
|
||||
outfile_names = [f"{args.tag}_class{i}_roc.png" for i in classes_to_plot]
|
||||
title_suffixes = [f"Class: {name}" for name in class_names]
|
||||
|
||||
# plot per class: all models on same axes
|
||||
out_json = {"tag": args.tag, "mode": args.mode, "head": args.head,
|
||||
"fusion_mode_filter": args.fusion_mode, "figures": []}
|
||||
|
||||
for k, cname, out_name, t_suffix in zip(classes_to_plot, class_names, outfile_names, title_suffixes):
|
||||
fig = plt.figure(figsize=(10, 8)); ax = fig.add_subplot(111)
|
||||
ax.plot([0,1],[0,1], linestyle="--", linewidth=1)
|
||||
ax.set_xlabel("False Positive Rate"); ax.set_ylabel("True Positive Rate")
|
||||
title_bits = [f"Combined ROC — {args.tag}", t_suffix, f"[{args.head}]"]
|
||||
if args.fusion_mode: title_bits.append(f"[{args.fusion_mode}]")
|
||||
ax.set_title(" — ".join(title_bits))
|
||||
|
||||
entries = []
|
||||
for label, curves in per_model:
|
||||
if k not in curves:
|
||||
continue
|
||||
c = curves[k]
|
||||
ax.plot(c["fpr"], c["tpr_mean"], linewidth=2,
|
||||
label=f"{label} (AUC {c['auc_mean']:.3f}±{c['auc_std']:.3f})")
|
||||
if args.shade:
|
||||
ax.fill_between(c["fpr"],
|
||||
np.maximum(c["tpr_mean"] - c["tpr_std"], 0),
|
||||
np.minimum(c["tpr_mean"] + c["tpr_std"], 1),
|
||||
alpha=0.10)
|
||||
entries.append({"label": label, "auc_mean": c["auc_mean"], "auc_std": c["auc_std"]})
|
||||
|
||||
ax.legend(loc="lower right")
|
||||
fig.tight_layout()
|
||||
|
||||
out_path = analysis_dir / out_name
|
||||
fig.savefig(out_path, dpi=160); plt.close(fig)
|
||||
|
||||
out_json["figures"].append({
|
||||
"class_index": k, "class_name": cname, "output_png": str(out_path),
|
||||
"models": entries
|
||||
})
|
||||
|
||||
# metadata file
|
||||
meta_path = analysis_dir / f"{args.tag}_perclass_summary.json"
|
||||
meta_path.write_text(json.dumps(out_json, indent=2), encoding="utf-8")
|
||||
print(f"Wrote figures + {meta_path}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,447 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Recompute per-fold ROC plots for a completed multifold run using the saved
|
||||
best checkpoints instead of the final epoch.
|
||||
|
||||
Example:
|
||||
python scripts/rebuild_run_best_plots.py \
|
||||
--run-dir analysis_data/1029_Baseline_Balanced_Resnet/1029_Baseline_Balanced_Resnet_20251029_163906 \
|
||||
--head image
|
||||
"""
|
||||
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import matplotlib
|
||||
import sys
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt # noqa: E402
|
||||
import numpy as np # noqa: E402
|
||||
import pandas as pd # noqa: E402
|
||||
import torch # noqa: E402
|
||||
from sklearn.metrics import auc, roc_auc_score, roc_curve # noqa: E402
|
||||
|
||||
from classes import build_papila_clinical # noqa: E402
|
||||
from classes.hypertower import HyperTower # noqa: E402
|
||||
|
||||
try: # Allow checkpoints that stored pandas DataFrames in their args.
|
||||
from torch.serialization import add_safe_globals # type: ignore
|
||||
|
||||
add_safe_globals([pd.DataFrame])
|
||||
except (ImportError, AttributeError):
|
||||
pass
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
ap = argparse.ArgumentParser(description="Rebuild ROC plots for an existing multifold run.")
|
||||
ap.add_argument("--run-dir", required=True, type=Path, help="Path to the run directory under analysis_data.")
|
||||
ap.add_argument("--head", default="image", choices=["image", "fused", "metadata"], help="Which prediction head to plot.")
|
||||
ap.add_argument("--class-names", nargs="*", default=None, help="Optional class names to control plot labels.")
|
||||
ap.add_argument("--overwrite", action="store_true", help="Overwrite existing .npy probability dumps if present.")
|
||||
ap.add_argument(
|
||||
"--use-holdout",
|
||||
action="store_true",
|
||||
help="Evaluate checkpoints on the saved holdout set instead of the fold validation splits.",
|
||||
)
|
||||
return ap.parse_args()
|
||||
|
||||
|
||||
def load_cli_args(run_dir: Path) -> dict:
|
||||
cli_path = run_dir / "cli_args.json"
|
||||
if not cli_path.exists():
|
||||
raise FileNotFoundError(f"Missing cli_args.json in {run_dir}")
|
||||
with cli_path.open("r", encoding="utf-8") as fh:
|
||||
return json.load(fh)
|
||||
|
||||
|
||||
def load_summary(run_dir: Path) -> dict:
|
||||
summary_path = run_dir / "summary.json"
|
||||
if not summary_path.exists():
|
||||
raise FileNotFoundError(f"Missing summary.json in {run_dir}")
|
||||
with summary_path.open("r", encoding="utf-8") as fh:
|
||||
return json.load(fh)
|
||||
|
||||
|
||||
def prepare_clinical(cli_args: dict, run_dir: Path) -> tuple:
|
||||
clinical = build_papila_clinical(
|
||||
cli_args["image_dir"],
|
||||
cli_args["clinical_dir"],
|
||||
cli_args["label_col"],
|
||||
cli_args["cat_cols"],
|
||||
n_splits=cli_args["n_splits"],
|
||||
random_seed=cli_args["fold_seed"],
|
||||
)
|
||||
|
||||
holdout_path = run_dir / "holdout.csv"
|
||||
holdout_df = pd.read_csv(holdout_path) if holdout_path.exists() else None
|
||||
if holdout_df is not None:
|
||||
if cli_args["eval_mode"] == "binary":
|
||||
holdout_df = holdout_df[holdout_df[cli_args["label_col"]].isin([0, 1])].reset_index(drop=True)
|
||||
|
||||
join_cols = [c for c in holdout_df.columns if c in clinical.df.columns]
|
||||
if not join_cols:
|
||||
raise RuntimeError("Holdout CSV found but no overlapping columns with clinical dataframe.")
|
||||
marker = holdout_df.assign(_holdout_marker=1)
|
||||
merged = clinical.df.merge(marker, on=join_cols, how="left")
|
||||
train_df = merged[merged["_holdout_marker"].isna()].drop(columns=["_holdout_marker"]).reset_index(drop=True)
|
||||
clinical.frames = [train_df.copy()]
|
||||
clinical.df = train_df.copy()
|
||||
clinical._infer_or_validate_feature_types()
|
||||
clinical._compute_numeric_stats()
|
||||
clinical._build_cat_maps()
|
||||
clinical._compute_feature_dim()
|
||||
clinical._build_kfold_indices()
|
||||
return clinical, holdout_df
|
||||
|
||||
|
||||
def build_ht_args(cli_args: dict, fold: int, run_dir: Path, models_dir: Path, holdout_df):
|
||||
# Copy of the training-time namespace so HyperTower can be re-instantiated.
|
||||
return SimpleNamespace(
|
||||
image_dir=cli_args["image_dir"],
|
||||
clinical_dir=cli_args["clinical_dir"],
|
||||
label_col=cli_args["label_col"],
|
||||
cat_cols=cli_args["cat_cols"],
|
||||
batch_size=cli_args["batch_size"],
|
||||
epochs=cli_args["epochs"],
|
||||
lr=cli_args["lr"],
|
||||
num_classes=cli_args["num_classes"],
|
||||
img_augment=cli_args.get("img_augment", True),
|
||||
focal_gamma=cli_args.get("focal_gamma", 0.0),
|
||||
eval_mode=cli_args["eval_mode"],
|
||||
fold=fold,
|
||||
run_dir=str(run_dir),
|
||||
models_dir=str(models_dir),
|
||||
backbone=cli_args["backbone"],
|
||||
freeze_ratio=cli_args["freeze_ratio"],
|
||||
fusion_mode=cli_args["fusion_mode"],
|
||||
use_se=cli_args.get("use_se", True),
|
||||
se_reduction=cli_args.get("se_reduction", 16),
|
||||
se_pre_norm=cli_args.get("se_pre_norm", True),
|
||||
se_where=cli_args.get("se_where", "bridge"),
|
||||
se_reduction_tower=cli_args.get("se_reduction_tower", cli_args.get("se_reduction", 16)),
|
||||
se_pre_norm_tower=cli_args.get("se_pre_norm_tower", cli_args.get("se_pre_norm", True)),
|
||||
warmup_tower_epochs=cli_args.get("warmup_tower_epochs", 0),
|
||||
warmup_fused_epochs=cli_args.get("warmup_fused_epochs", 0),
|
||||
gradual_thaw=cli_args.get("gradual_thaw", False),
|
||||
thaw_phase_duration=cli_args.get("thaw_phase_duration", 5),
|
||||
thaw_ratio=cli_args.get("thaw_ratio", 0.33),
|
||||
thaw_target=cli_args.get("thaw_target", "image"),
|
||||
thaw_start_epoch=cli_args.get("thaw_start_epoch", -1),
|
||||
initial_freeze=cli_args.get("initial_freeze", False),
|
||||
bcd_prob=0.5,
|
||||
bcd_p0=0.20,
|
||||
bcd_min=0.05,
|
||||
bcd_max=0.30,
|
||||
bcd_k=0.4,
|
||||
bcd_metric="auc",
|
||||
bcd_alpha_batch=0.2,
|
||||
bcd_alpha_tower=0.3,
|
||||
bcd_explore_floor=0.15,
|
||||
aux_img=0.05,
|
||||
aux_md=0.05,
|
||||
aux_detach=True,
|
||||
ema_alpha=0.9,
|
||||
entropy_ema=0.7,
|
||||
early_stop=cli_args.get("early_stop", False),
|
||||
early_metric=cli_args.get("early_metric"),
|
||||
early_mode=cli_args.get("early_mode", "auto"),
|
||||
early_patience=cli_args.get("early_patience", 7),
|
||||
early_min_delta=cli_args.get("early_min_delta", 0.0),
|
||||
checkpoint_best=cli_args.get("checkpoint_best", False),
|
||||
holdout_df=holdout_df,
|
||||
img_crop_manifest=cli_args.get("img_crop_manifest"),
|
||||
img_crop_weights=cli_args.get("img_crop_weights"),
|
||||
img_crop_normalize=cli_args.get("img_crop_normalize"),
|
||||
img_crop_threshold=cli_args.get("img_crop_threshold"),
|
||||
img_crop_scale=cli_args.get("img_crop_scale", 2.5),
|
||||
img_crop_size=cli_args.get("img_crop_size", 224),
|
||||
img_crop_cache=cli_args.get("img_crop_cache"),
|
||||
img_crop_tta=cli_args.get("img_crop_tta", False),
|
||||
img_crop_gt=cli_args.get("img_crop_gt", False),
|
||||
img_geometry_features=cli_args.get("img_geometry_features", False),
|
||||
balanced_sampler=cli_args.get("balanced_sampler", False),
|
||||
)
|
||||
|
||||
|
||||
def collect_logits(ht, loader):
|
||||
"""Mirror Multifold.eval_collect_logits but for a provided loader."""
|
||||
device = ht.device
|
||||
ht.img_tower.eval()
|
||||
ht.md_tower.eval()
|
||||
outputs = []
|
||||
with torch.no_grad():
|
||||
if ht.mode == "vote":
|
||||
ht.head_img.eval()
|
||||
ht.head_md.eval()
|
||||
ht.vote.eval()
|
||||
else:
|
||||
ht.bridge.eval()
|
||||
|
||||
for batch in loader:
|
||||
if len(batch) == 4:
|
||||
imgs, metas, geometry, labels = batch
|
||||
else:
|
||||
imgs, metas, labels = batch
|
||||
geometry = None
|
||||
imgs = imgs.to(device)
|
||||
metas = metas.to(device)
|
||||
labels = labels.to(device)
|
||||
if geometry is not None and geometry.numel() > 0:
|
||||
geometry = geometry.to(device)
|
||||
else:
|
||||
geometry = None
|
||||
if ht.mode == "vote":
|
||||
img_feats = ht.img_tower(imgs, geometry)
|
||||
md_feats = ht.md_tower(metas)
|
||||
out_img = ht.head_img(img_feats)
|
||||
out_md = ht.head_md(md_feats)
|
||||
out_fused = ht.vote(out_img, out_md)
|
||||
else:
|
||||
img_feats = ht.img_tower(imgs, geometry)
|
||||
md_feats = ht.md_tower(metas)
|
||||
result = ht.bridge(img_feats, md_feats)
|
||||
if isinstance(result, tuple):
|
||||
out_fused, out_img, out_md = result
|
||||
else:
|
||||
out_fused, out_img, out_md = result, None, None
|
||||
|
||||
outputs.append(
|
||||
(
|
||||
labels.detach().cpu().numpy(),
|
||||
torch.softmax(out_fused, dim=1).detach().cpu().numpy() if out_fused is not None else None,
|
||||
torch.softmax(out_img, dim=1).detach().cpu().numpy() if out_img is not None else None,
|
||||
torch.softmax(out_md, dim=1).detach().cpu().numpy() if out_md is not None else None,
|
||||
)
|
||||
)
|
||||
|
||||
if not outputs:
|
||||
return np.array([]), None, None, None
|
||||
|
||||
y_all, pf, pi, pm = zip(*outputs)
|
||||
y_true = np.concatenate(y_all, axis=0)
|
||||
probs_f = np.concatenate([p for p in pf if p is not None], axis=0) if any(p is not None for p in pf) else None
|
||||
probs_i = np.concatenate([p for p in pi if p is not None], axis=0) if any(p is not None for p in pi) else None
|
||||
probs_m = np.concatenate([p for p in pm if p is not None], axis=0) if any(p is not None for p in pm) else None
|
||||
return y_true, probs_f, probs_i, probs_m
|
||||
|
||||
|
||||
def compute_per_class_curves(y_true, probs):
|
||||
if probs is None:
|
||||
return {}
|
||||
num_classes = probs.shape[1]
|
||||
curves = {}
|
||||
for k in range(num_classes):
|
||||
y_bin = (y_true == k).astype(np.uint8)
|
||||
fpr, tpr, _ = roc_curve(y_bin, probs[:, k])
|
||||
curves[k] = {"fpr": fpr, "tpr": tpr, "auc": auc(fpr, tpr) if len(fpr) > 1 else np.nan}
|
||||
return curves
|
||||
|
||||
|
||||
def choose_head_probs(head: str, probs_f, probs_i, probs_m):
|
||||
if head == "fused":
|
||||
return probs_f
|
||||
if head == "metadata":
|
||||
return probs_m
|
||||
return probs_i
|
||||
|
||||
|
||||
def ensure_binary_slice(y_true, *arrays):
|
||||
mask = np.isin(y_true, [0, 1])
|
||||
filtered = [y_true[mask]]
|
||||
for arr in arrays:
|
||||
if arr is None:
|
||||
filtered.append(None)
|
||||
else:
|
||||
filtered.append(arr[mask])
|
||||
return filtered
|
||||
|
||||
|
||||
def plot_overlays(per_fold_curves, out_dir: Path, class_names: list[str], head: str, suffix: str = ""):
|
||||
keys = sorted({k for _, curves in per_fold_curves for k in curves.keys()})
|
||||
if not keys:
|
||||
return
|
||||
name_map = {k: (class_names[k] if k < len(class_names) else f"class_{k}") for k in keys}
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for k in keys:
|
||||
fig = plt.figure(figsize=(10, 8))
|
||||
ax = fig.add_subplot(111)
|
||||
ax.plot([0, 1], [0, 1], linestyle="--", linewidth=1, color="grey")
|
||||
for fold_idx, curves in per_fold_curves:
|
||||
if k not in curves:
|
||||
continue
|
||||
fpr = curves[k]["fpr"]
|
||||
tpr = curves[k]["tpr"]
|
||||
auc_val = curves[k]["auc"]
|
||||
label = f"Fold {fold_idx} (AUC {auc_val:.3f})" if auc_val == auc_val else f"Fold {fold_idx}"
|
||||
ax.plot(fpr, tpr, linewidth=1.5, label=label)
|
||||
ax.set_xlabel("False Positive Rate")
|
||||
ax.set_ylabel("True Positive Rate")
|
||||
ax.set_title(f"{head} head — {name_map[k]} ROC per fold")
|
||||
ax.legend(loc="lower right")
|
||||
fig.tight_layout()
|
||||
safe_name = name_map[k].replace(" ", "_")
|
||||
suffix_str = suffix if suffix else ""
|
||||
fig.savefig(out_dir / f"roc_{head}_{safe_name}_perfold{suffix_str}.png", dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def plot_mean_sd(per_fold_curves, out_dir: Path, class_names: list[str], head: str, suffix: str = ""):
|
||||
keys = sorted({k for _, curves in per_fold_curves for k in curves.keys()})
|
||||
if not keys:
|
||||
return
|
||||
grid = np.linspace(0, 1, 501)
|
||||
fig = plt.figure(figsize=(10, 8))
|
||||
ax = fig.add_subplot(111)
|
||||
ax.plot([0, 1], [0, 1], linestyle="--", linewidth=1, color="grey")
|
||||
for k in keys:
|
||||
tprs = []
|
||||
aucs = []
|
||||
for _, curves in per_fold_curves:
|
||||
if k not in curves:
|
||||
continue
|
||||
fpr = curves[k]["fpr"]
|
||||
tpr = curves[k]["tpr"]
|
||||
aucs.append(curves[k]["auc"])
|
||||
tprs.append(np.interp(grid, fpr, tpr))
|
||||
if not tprs:
|
||||
continue
|
||||
tprs = np.vstack(tprs)
|
||||
mean = tprs.mean(axis=0)
|
||||
std = tprs.std(axis=0)
|
||||
label = class_names[k] if k < len(class_names) else f"class_{k}"
|
||||
label = f"{label} (AUC {np.nanmean(aucs):.3f}±{np.nanstd(aucs):.3f})"
|
||||
ax.plot(grid, mean, linewidth=2, label=label)
|
||||
ax.fill_between(grid, np.maximum(mean - std, 0), np.minimum(mean + std, 1), alpha=0.15)
|
||||
ax.set_xlabel("False Positive Rate")
|
||||
ax.set_ylabel("True Positive Rate")
|
||||
ax.set_title(f"Mean OVR ROC (±1 SD) — {head} head")
|
||||
ax.legend(loc="lower right")
|
||||
fig.tight_layout()
|
||||
suffix_str = suffix if suffix else ""
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out_dir / f"roc_{head}_mean_ovr{suffix_str}.png", dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
run_dir = args.run_dir.resolve()
|
||||
cli_args = load_cli_args(run_dir)
|
||||
summary = load_summary(run_dir)
|
||||
|
||||
class_names = (
|
||||
args.class_names
|
||||
if args.class_names
|
||||
else (cli_args.get("class_names") or (["Healthy", "Glaucoma"] if cli_args["eval_mode"] == "binary" else ["Healthy", "Glaucoma", "Suspect"]))
|
||||
)
|
||||
|
||||
shortname = cli_args.get("shortname") or run_dir.parent.name
|
||||
run_id = cli_args.get("run_id") or run_dir.name
|
||||
base_models_dir = Path("models") / shortname / run_id
|
||||
|
||||
clinical, holdout_df = prepare_clinical(cli_args, run_dir)
|
||||
if args.use_holdout and holdout_df is None:
|
||||
raise SystemExit("Holdout metrics requested but no holdout.csv found for this run.")
|
||||
|
||||
per_fold_curves = []
|
||||
fold_aucs = []
|
||||
head = args.head
|
||||
file_suffix = "_holdout" if args.use_holdout else ""
|
||||
|
||||
for fold_entry in summary.get("fold_metrics", []):
|
||||
fold_idx = int(fold_entry["fold"])
|
||||
best_epoch = fold_entry.get("best_epoch")
|
||||
if not best_epoch:
|
||||
print(f"[skip] Fold {fold_idx}: no best_epoch recorded.")
|
||||
continue
|
||||
|
||||
fold_models_dir = base_models_dir / f"fold{fold_idx}"
|
||||
best_checkpoint = fold_models_dir / "model_best.pt"
|
||||
if not best_checkpoint.exists():
|
||||
print(f"[warning] Fold {fold_idx}: missing model_best.pt at {best_checkpoint}")
|
||||
continue
|
||||
|
||||
ht_args = build_ht_args(cli_args, fold_idx, run_dir, fold_models_dir, holdout_df)
|
||||
ht = HyperTower(clinical, ht_args)
|
||||
for handler in list(ht.logger.handlers):
|
||||
handler.close()
|
||||
ht.logger.handlers = [logging.NullHandler()]
|
||||
train_log_path = Path("train.log")
|
||||
if train_log_path.exists() and train_log_path.stat().st_size == 0:
|
||||
try:
|
||||
train_log_path.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
state = torch.load(best_checkpoint, map_location=ht.device, weights_only=False)
|
||||
except TypeError:
|
||||
state = torch.load(best_checkpoint, map_location=ht.device)
|
||||
ht._restore_from_state(state)
|
||||
if args.use_holdout:
|
||||
eval_df = holdout_df.copy()
|
||||
else:
|
||||
_, eval_df = clinical.get_split_dfs(fold_idx)
|
||||
if cli_args["eval_mode"] == "binary":
|
||||
eval_df = eval_df[eval_df[cli_args["label_col"]].isin([0, 1])].reset_index(drop=True)
|
||||
if eval_df.empty:
|
||||
print(f"[warning] Fold {fold_idx}: evaluation dataframe is empty; skipping.")
|
||||
continue
|
||||
ht.test_loader = ht._make_loader_for_df(eval_df, is_train=False)
|
||||
|
||||
y_true, probs_f, probs_i, probs_m = collect_logits(ht, ht.test_loader)
|
||||
if cli_args["eval_mode"] == "binary":
|
||||
y_true, probs_f, probs_i, probs_m = ensure_binary_slice(y_true, probs_f, probs_i, probs_m)
|
||||
|
||||
head_probs = choose_head_probs(head, probs_f, probs_i, probs_m)
|
||||
if head_probs is None:
|
||||
print(f"[skip] Fold {fold_idx}: head '{head}' not available.")
|
||||
continue
|
||||
|
||||
if head_probs.shape[1] >= 2:
|
||||
head_probs = head_probs[:, :2]
|
||||
|
||||
if args.overwrite:
|
||||
base = run_dir / f"fold{fold_idx}{file_suffix}"
|
||||
np.save(f"{base}_y_true.npy", y_true)
|
||||
if probs_f is not None:
|
||||
np.save(f"{base}_probs_fused.npy", probs_f)
|
||||
if probs_i is not None:
|
||||
np.save(f"{base}_probs_img.npy", probs_i)
|
||||
if probs_m is not None:
|
||||
np.save(f"{base}_probs_md.npy", probs_m)
|
||||
|
||||
curves = compute_per_class_curves(y_true, head_probs)
|
||||
per_fold_curves.append((fold_idx, curves))
|
||||
try:
|
||||
if head_probs.shape[1] > 2:
|
||||
fold_auc = roc_auc_score(y_true, head_probs, multi_class="ovr", average="macro")
|
||||
else:
|
||||
target_scores = head_probs[:, 1] if head_probs.shape[1] > 1 else head_probs[:, 0]
|
||||
fold_auc = roc_auc_score(y_true, target_scores)
|
||||
fold_aucs.append(fold_auc)
|
||||
print(f"[info] Fold {fold_idx}: best epoch {best_epoch}, AUC={fold_auc:.4f}")
|
||||
except Exception:
|
||||
print(f"[warning] Fold {fold_idx}: unable to compute AUC.")
|
||||
|
||||
if not per_fold_curves:
|
||||
raise SystemExit("No folds processed; nothing to plot.")
|
||||
|
||||
plots_dir = run_dir / "plots"
|
||||
plot_overlays(per_fold_curves, plots_dir, class_names, head, file_suffix)
|
||||
plot_mean_sd(per_fold_curves, plots_dir, class_names, head, file_suffix)
|
||||
|
||||
if fold_aucs:
|
||||
print(f"[info] {head} head mean AUC across folds: {np.mean(fold_aucs):.4f} ± {np.std(fold_aucs):.4f}")
|
||||
print(f"Plots regenerated under {plots_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user