pre-refactor 041426

v3/scripts/output_analysis/plot_cnn_backbone_comparison.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+"""
+Bar plot comparing CNN standalone vs HyperTower image_only AUC per backbone,
+with PAPILA paper reference lines.
+
+Usage:
+  python -m v3.scripts.output_analysis.plot_cnn_backbone_comparison \
+      --results-dir v3/results/phase1 \
+      --output v3/results/phase1/cnn_backbone_comparison.png
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from sklearn.metrics import roc_auc_score
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
+
+
+BACKBONES = ["densenet121", "vgg16", "mobilenet_v2", "inception_v3", "resnet50"]
+BACKBONE_LABELS = {
+    "densenet121":  "DenseNet121",
+    "vgg16":        "VGG16",
+    "mobilenet_v2": "MobileNetV2",
+    "inception_v3": "Inception V3",
+    "resnet50":     "ResNet50",
+}
+
+# Per-backbone paper AUCs (binary, Test #2, PAPILA 2022)
+PAPER_AUC = {
+    "densenet121":  0.80,
+    "vgg16":        0.84,
+    "mobilenet_v2": 0.75,
+    "inception_v3": 0.78,
+    "resnet50":     0.78,
+}
+PAPER_STD = {
+    "densenet121":  0.05,
+    "vgg16":        0.02,
+    "mobilenet_v2": 0.06,
+    "inception_v3": 0.08,
+    "resnet50":     0.07,
+}
+
+COLOURS = {
+    "cnn":       "#4878CF",
+    "ht":        "#D65F5F",
+    "paper_ref": "black",
+}
+
+
+def load_cnn_fold_aucs(results_dir: Path, backbone: str) -> list[float]:
+    fpath = results_dir / f"cnn_{backbone}" / "fold_metrics.csv"
+    if not fpath.exists():
+        print(f"  WARNING: missing {fpath}")
+        return []
+    df = pd.read_csv(fpath)
+    return df["auc"].tolist()
+
+
+def load_ht_fold_aucs(results_dir: Path, backbone: str, n_folds: int = 5) -> list[float]:
+    aucs = []
+    for fold in range(n_folds):
+        fold_dir = results_dir / "imageonly_ht" / backbone / "binary" / "single" / f"fold{fold}"
+        y_path  = fold_dir / "test_y_true.npy"
+        p_path  = fold_dir / "test_probs_fused.npy"
+        if not (y_path.exists() and p_path.exists()):
+            print(f"  WARNING: missing predictions for {backbone} fold{fold}")
+            continue
+        y  = np.load(y_path)
+        pr = np.load(p_path)
+        if len(np.unique(y)) < 2:
+            print(f"  WARNING: single-class test set for {backbone} fold{fold}, skipping")
+            continue
+        aucs.append(float(roc_auc_score(y, pr[:, 1])))
+    return aucs
+
+
+def plot(cnn_data: dict, ht_data: dict, output: Path):
+    n = len(BACKBONES)
+    x = np.arange(n)
+    group_width = 0.7
+    bar_w = group_width / 2 * 0.88
+    offsets = [-group_width / 4, group_width / 4]
+
+    fig, ax = plt.subplots(figsize=(10, 5.5))
+
+    for bi, backbone in enumerate(BACKBONES):
+        for si, (tag, data, colour) in enumerate([
+            ("CNN standalone", cnn_data, COLOURS["cnn"]),
+            ("HyperTower (image only)", ht_data, COLOURS["ht"]),
+        ]):
+            aucs = data.get(backbone, [])
+            if not aucs:
+                continue
+            xpos = bi + offsets[si]
+            mean, std = np.mean(aucs), np.std(aucs)
+            ax.bar(
+                xpos, mean, width=bar_w,
+                color=colour, alpha=0.80,
+                label=tag if bi == 0 else "_nolegend_",
+            )
+            ax.errorbar(
+                xpos, mean, yerr=std,
+                fmt="none", color="black", capsize=4, linewidth=1.2,
+            )
+
+        # Paper reference line spanning this backbone's group
+        paper_val = PAPER_AUC.get(backbone)
+        if paper_val is not None:
+            lw = group_width / 2 + bar_w / 2
+            label = "PAPILA paper" if bi == 0 else "_nolegend_"
+            ax.hlines(
+                paper_val,
+                bi - group_width / 2, bi + group_width / 2,
+                colors=COLOURS["paper_ref"], linestyles=":", linewidths=1.8,
+                label=label,
+            )
+
+    ax.set_xticks(x)
+    ax.set_xticklabels([BACKBONE_LABELS[b] for b in BACKBONES], fontsize=11)
+    ax.set_ylabel("AUC (ROC)", fontsize=11)
+    ax.set_title("Phase 1: CNN backbone AUC — standalone vs HyperTower (image only)", fontsize=12)
+    ax.set_ylim(0.45, 1.02)
+    ax.axhline(0.5, color="grey", linestyle="--", linewidth=0.8, alpha=0.4)
+    ax.grid(axis="y", alpha=0.3, linestyle="--")
+    ax.legend(loc="lower right", fontsize=10, framealpha=0.9)
+    fig.tight_layout()
+
+    output.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(output, dpi=180)
+    plt.close(fig)
+    print(f"Saved: {output}")
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--results-dir", default="v3/results/phase1")
+    ap.add_argument("--output", default=None)
+    args = ap.parse_args()
+
+    results_dir = Path(args.results_dir)
+    output = Path(args.output) if args.output else results_dir / "cnn_backbone_comparison.png"
+
+    cnn_data = {b: load_cnn_fold_aucs(results_dir, b) for b in BACKBONES}
+    ht_data  = {b: load_ht_fold_aucs(results_dir, b)  for b in BACKBONES}
+
+    plot(cnn_data, ht_data, output)
+
+    # Summary table
+    print(f"\n{'Backbone':<16}  {'CNN standalone':>18}  {'HT image_only':>18}  {'Paper':>12}")
+    print("-" * 72)
+    for b in BACKBONES:
+        cnn_aucs = cnn_data[b]
+        ht_aucs  = ht_data[b]
+        cnn_str  = f"{np.mean(cnn_aucs):.3f} ± {np.std(cnn_aucs):.3f}" if cnn_aucs else "—"
+        ht_str   = f"{np.mean(ht_aucs):.3f} ± {np.std(ht_aucs):.3f}"  if ht_aucs  else "—"
+        p_str    = f"{PAPER_AUC[b]:.2f} ± {PAPER_STD[b]:.2f}"
+        print(f"{BACKBONE_LABELS[b]:<16}  {cnn_str:>18}  {ht_str:>18}  {p_str:>12}")
+
+
+if __name__ == "__main__":
+    main()