2026001

scripts/visualizations/figure1.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""figure1.py — Sample CT images from the IQ-OTH/NCCD dataset, one per patient.
+Usage: python scripts/visualizations/figure1.py [--tag TAG]"""
+
+import os, sys, csv, argparse
+import matplotlib; matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from PIL import Image
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__)))))
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__))))
+DATASET = os.path.join(os.path.dirname(ROOT), "The IQ-OTHNCCD lung cancer dataset")
+MANIFEST = os.path.join(ROOT, "results", "simple_patient_manifest.csv")
+PLOTS_DIR = os.path.join(ROOT, "plots")
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--tag", default="", help="Append tag to filename")
+ap.add_argument("--manifest", default=MANIFEST, help="Patient manifest CSV")
+args = ap.parse_args()
+tag = f"_{args.tag}" if args.tag else ""
+
+# Load manifest to get per-patient images
+patients = {"Benign": [], "Malignant": [], "Normal": []}
+with open(args.manifest, newline="") as f:
+    reader = csv.DictReader(f)
+    img_col = "confirmed_images" if "confirmed_images" in reader.fieldnames else "images"
+    for row in reader:
+        cls = row.get("class", "")
+        if cls in patients:
+            imgs = row[img_col].split(";")
+            if imgs:
+                patients[cls].append((row["patient_id"], imgs[0]))  # first image per patient
+
+CLASS_DIR = {"Benign": "Bengin cases", "Malignant": "Malignant cases",
+             "Normal": "Normal cases"}
+N_EXAMPLES = 3
+
+fig, axes = plt.subplots(3, N_EXAMPLES, figsize=(8, 9))
+for row, (cls_label, cls_dir) in enumerate(CLASS_DIR.items()):
+    # Pick first N_EXAMPLES patients for this class
+    selected = patients[cls_label][:N_EXAMPLES]
+    for col, (pid, short_name) in enumerate(selected):
+        ax = axes[row, col]
+        # Convert short name back to original filename
+        prefix = short_name[0]
+        num = int(short_name.split("_")[1])
+        cls_map = {"B": ("Bengin cases", "Bengin"), "M": ("Malignant cases", "Malignant"),
+                   "N": ("Normal cases", "Normal")}
+        dir_name, file_prefix = cls_map[prefix]
+        fname = f"{file_prefix} case ({num}).jpg"
+        img_path = os.path.join(DATASET, dir_name, fname)
+        try:
+            img = Image.open(img_path).convert("L")
+            ax.imshow(img, cmap="gray")
+        except Exception as e:
+            ax.text(0.5, 0.5, f"error: {e}", ha="center", va="center", fontsize=7)
+        ax.set_xticks([]); ax.set_yticks([])
+        if col == 0:
+            ax.set_ylabel(cls_label, fontsize=10, rotation=0,
+                          labelpad=20, va="center")
+
+fig.suptitle("Figure 1 — Sample images from the IQ-OTH/NCCD dataset",
+             fontsize=12, y=1.02)
+plt.tight_layout(rect=[0, 0, 1, 0.97])
+out = os.path.join(PLOTS_DIR, "figure1", f"figure1{tag}.png")
+os.makedirs(os.path.dirname(out), exist_ok=True)
+plt.savefig(out, dpi=150)
+plt.close()
+print(f"Saved → {out}")

scripts/visualizations/figure3_s1_s4.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+"""Combined CV-accuracy-vs-features curves for Figures 3, S1, and S4.
+
+These three figures all compute the same per-model curve — RF importance
+ranking + a per-nfeatures gamma grid at seed 20 — differing only in which
+models and which split they show:
+
+    Figure 3  : VGG16, image-level      (1 panel)   == panel (a) of S1
+    Figure S1 : all 5 models, image     (5 panels)
+    Figure S4 : all 5 models, patient   (5 panels)
+
+Computing them together runs each (model, split) curve once (5 image + 5
+patient = 10 curves) and caches them, instead of recomputing VGG16's image
+curve for both Figure 3 and S1. The cache is incremental: dropping a model's
+entries (e.g. after a feature change) recomputes only that model.
+
+Usage:
+    conda activate fundus_imaging
+    python scripts/visualizations/figure3_s1_s4.py                # all three
+    python scripts/visualizations/figure3_s1_s4.py --only s4      # just Fig S4
+    python scripts/visualizations/figure3_s1_s4.py --from-cache   # re-render only
+    python scripts/visualizations/figure3_s1_s4.py --force        # recompute all
+"""
+
+import os
+import sys
+import json
+import argparse
+import numpy as np
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from tqdm import tqdm
+
+os.environ.setdefault("OMP_NUM_THREADS", "1")
+os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
+os.environ.setdefault("MKL_NUM_THREADS", "1")
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__)))))
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+# Shared results-cache tier (alongside classification_runs.json). These curves
+# are model-selection output (best nfeat/gamma/cv), not plotting scaffolding, so
+# they live in scripts/cache/ rather than under visualizations/.
+CACHE_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "cache")
+PLOTS_DIR = os.path.join(ROOT, "plots")
+
+from classes import PatientLeakageClassifier
+
+# Panel order matches the published supplementary figures.
+MODELS = ["VGG16", "MobileNetV2", "DenseNet121", "ResNet50", "EfficientNetB1"]
+SEED = 20
+DATA_PATH = os.path.join(CACHE_DIR, "cv_curves.json")
+MANIFEST = os.path.join(ROOT, "results", "simple_patient_manifest.csv")
+
+# 16-point log grid of feature counts (each model caps at its own max dim).
+NFEATS = np.unique(np.round(np.logspace(1.5, 5, 16)).astype(int)).tolist()
+GAMMA_LOGSPACE = (-1.5, 1, 7)
+PANELS = ["(a)", "(b)", "(c)", "(d)", "(e)"]
+
+
+def key(model, split):
+    return f"{model}::{split}"
+
+
+def load_cache():
+    if os.path.exists(DATA_PATH):
+        with open(DATA_PATH) as f:
+            return json.load(f)
+    return {}
+
+
+def compute(cache, splits, n_jobs, manifest):
+    """Fill any missing (model, split) curves in the cache; return updated cache."""
+    todo = [(m, s) for s in splits for m in MODELS if key(m, s) not in cache]
+    if not todo:
+        print(f"Cache complete for requested figures ({len(cache)} curves).")
+        return cache
+    clf = PatientLeakageClassifier(manifest, os.path.join(ROOT, "features"),
+                                   n_jobs=n_jobs)
+    for m, s in tqdm(todo, desc="Computing missing curves"):
+        cache[key(m, s)] = clf.cv_curve(m, SEED, s, nfeatures_list=NFEATS,
+                                        gamma_logspace=GAMMA_LOGSPACE)
+    os.makedirs(CACHE_DIR, exist_ok=True)
+    with open(DATA_PATH, "w") as f:
+        json.dump(cache, f, indent=2)
+    print(f"Computed {len(todo)} curves; cache now {len(cache)}.")
+    return cache
+
+
+def _draw_curve(ax, curve, color, ylim, title=None):
+    # Keys are ints on a fresh compute but strings once round-tripped through
+    # JSON; iterate items so either works.
+    items = sorted((int(n), v) for n, v in curve["curves"].items())
+    nfeats = [n for n, _ in items]
+    accs = [v * 100 for _, v in items]
+    ax.plot(nfeats, accs, "o-", color=color, markersize=4, linewidth=1.2)
+    ax.set_xscale("log")
+    ax.set_xlabel("Number of selected features")
+    ax.set_ylabel("CV accuracy (%)")
+    ax.set_ylim(*ylim)
+    ax.grid(alpha=0.3)
+    bn, ba = curve["best_nfeat"], curve["best_cv"] * 100
+    ax.axvline(bn, color="red", linestyle="--", linewidth=0.8, alpha=0.6)
+    ax.plot(bn, ba, "r*", markersize=11)
+    ax.annotate(f"n={bn}\n{ba:.2f}%", (bn, ba), fontsize=8,
+                xytext=(10, -10), textcoords="offset points", color="red")
+    if title:
+        ax.set_title(title, fontsize=10)
+
+
+def render_fig3(cache, tag):
+    fig, ax = plt.subplots(figsize=(8, 5))
+    _draw_curve(ax, cache[key("VGG16", "image")], "#D62728", (92, 100))
+    ax.set_title("Figure 3 — VGG16 Image-level CV accuracy vs number of "
+                 "selected features", fontsize=12)
+    _save(fig, "figure3", f"figure3{tag}.png")
+
+
+def render_grid(cache, split, color, ylim, suptitle, subdir, fname):
+    fig, axes = plt.subplots(2, 3, figsize=(14, 9))
+    axes = axes.flatten()
+    for i, model in enumerate(MODELS):
+        _draw_curve(axes[i], cache[key(model, split)], color, ylim,
+                    title=f"{PANELS[i]} {model}")
+    axes[5].set_visible(False)
+    fig.suptitle(suptitle, fontsize=13, y=1.01)
+    _save(fig, subdir, fname, tight=True)
+
+
+def _save(fig, subdir, fname, tight=False):
+    plt.tight_layout()
+    out = os.path.join(PLOTS_DIR, subdir, fname)
+    os.makedirs(os.path.dirname(out), exist_ok=True)
+    fig.savefig(out, dpi=150, bbox_inches="tight" if tight else None)
+    plt.close(fig)
+    print(f"Saved → {out}")
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Combined Figures 3, S1 & S4.")
+    ap.add_argument("--only", choices=["3", "s1", "s4"],
+                    help="Render only one figure (still computes its curves).")
+    ap.add_argument("--from-cache", action="store_true",
+                    help="Re-render from cache without computing.")
+    ap.add_argument("--force", action="store_true",
+                    help="Recompute all curves, ignoring the cache.")
+    ap.add_argument("--n-jobs", type=int, default=6)
+    ap.add_argument("--manifest", default=MANIFEST)
+    ap.add_argument("--tag", default="")
+    args = ap.parse_args()
+    tag = f"_{args.tag}" if args.tag else ""
+
+    # Which splits are needed for the requested figure(s)?
+    need_image = args.only in (None, "3", "s1")
+    need_patient = args.only in (None, "s4")
+    splits = (["image"] if need_image else []) + (["patient"] if need_patient else [])
+
+    cache = {} if args.force else load_cache()
+    if args.from_cache:
+        print(f"Loaded {len(cache)} cached curves ← {DATA_PATH}")
+    else:
+        cache = compute(cache, splits, args.n_jobs, args.manifest)
+
+    if args.only in (None, "3"):
+        render_fig3(cache, tag)
+    if args.only in (None, "s1"):
+        render_grid(cache, "image", "#D62728", (90, 100),
+                    "Figure S1 — Image-level CV accuracy vs number of selected "
+                    "features", "figure_s1", f"figure_s1{tag}.png")
+    if args.only in (None, "s4"):
+        render_grid(cache, "patient", "#2C7BB6", (82, 95),
+                    "Figure S4 — Patient-level CV accuracy vs number of selected "
+                    "features (C=10)", "figure_s4", f"figure_s4{tag}.png")
+    print("DONE")
+
+
+if __name__ == "__main__":
+    main()

scripts/visualizations/figure4_6.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""Combined 20-seed classification for Figures 4 and 6 — one compute pass.
+
+Usage:
+    python scripts/visualizations/figure4_6.py              # both figures
+    python scripts/visualizations/figure4_6.py --only 4     # Fig 4 only
+    python scripts/visualizations/figure4_6.py --only 6     # Fig 6 only
+    python scripts/visualizations/figure4_6.py --from-cache # use saved data
+"""
+
+import os, sys, json, argparse
+import numpy as np
+import matplotlib; matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from matplotlib.patches import Patch
+from tqdm import tqdm
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from classes import PatientLeakageClassifier
+
+PLOTS_DIR = os.path.join(ROOT, "plots")
+RESULTS_DIR = os.path.join(ROOT, "results")
+# Shared classification-runs cache, owned by scripts/classification.py. Both
+# scripts fill it incrementally (same model/seed/split keys), merge-safe.
+CACHE_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "cache")
+MODELS = ["VGG16", "DenseNet121", "EfficientNetB1", "MobileNetV2", "ResNet50"]
+SEEDS = list(range(1, 21))
+DATA_PATH = os.path.join(CACHE_DIR, "classification_runs.json")
+MANIFEST = os.path.join(RESULTS_DIR, "simple_patient_manifest.csv")
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--only", default=None, choices=["4", "6"])
+ap.add_argument("--from-cache", action="store_true")
+ap.add_argument("--force", action="store_true", help="Recompute all runs, ignoring cache.")
+ap.add_argument("--manifest", default=MANIFEST)
+ap.add_argument("--tag", default="")
+args = ap.parse_args()
+tag = f"_{args.tag}" if args.tag else ""
+
+# Run or load. The cache is a flat list of per-run dicts; compute only the
+# (model, seed, split) combos it's missing, so dropping a stale model's rows
+# (e.g. ResNet50 after a feature change) recomputes just that model.
+all_data = []
+if os.path.exists(DATA_PATH) and not args.force:
+    with open(DATA_PATH) as f: all_data = json.load(f)
+
+if args.from_cache:
+    print(f"Loaded {len(all_data)} cached runs ← {DATA_PATH}")
+else:
+    have = {(r["model"], r["seed"], r["split_type"]) for r in all_data}
+    todo = [(m, s, st) for m in MODELS for s in SEEDS
+            for st in ("image", "patient") if (m, s, st) not in have]
+    if todo:
+        clf = PatientLeakageClassifier(args.manifest, os.path.join(ROOT, "features"), n_jobs=6)
+        for m, s, st in tqdm(todo, desc="Computing missing runs"):
+            all_data.append(clf.run(m, s, st))
+        os.makedirs(CACHE_DIR, exist_ok=True)
+        with open(DATA_PATH, "w") as f: json.dump(all_data, f, indent=2)
+        print(f"Computed {len(todo)} missing runs; cache now {len(all_data)}")
+    else:
+        print(f"Cache complete ({len(all_data)} runs); nothing to compute.")
+
+# Build accs. Only plot models that actually have data, so a partially filled
+# cache (e.g. ResNet50 dropped pending recompute) still renders without error.
+accs = {m: {"image": [], "patient": []} for m in MODELS}
+for r in all_data:
+    if r["model"] in accs: accs[r["model"]][r["split_type"]].append(r["test"])
+PLOT_MODELS = [m for m in MODELS if accs[m]["image"] or accs[m]["patient"]]
+missing = [m for m in MODELS if m not in PLOT_MODELS]
+if missing:
+    print(f"WARNING: no cached runs for {missing}; run without --from-cache "
+          "to compute them. Plotting remaining models only.")
+
+# Figure 4: image-level only
+if args.only is None or args.only == "4":
+    fig, ax = plt.subplots(figsize=(8, 5))
+    pos = list(range(1, len(PLOT_MODELS) + 1))
+    bp = ax.boxplot([accs[m]["image"] for m in PLOT_MODELS], positions=pos,
+                    widths=0.5, patch_artist=True, showfliers=True,
+                    flierprops=dict(marker='o', markersize=3))
+    for i, b in enumerate(bp['boxes']):
+        b.set_facecolor('#4C9BD4')
+        ax.annotate(f"{np.median(accs[PLOT_MODELS[i]]['image']):.3f}",
+                    (pos[i], np.median(accs[PLOT_MODELS[i]]['image'])),
+                    fontsize=6, ha='center', va='bottom')
+    ax.set_xticks(pos); ax.set_xticklabels(PLOT_MODELS)
+    ax.set_ylabel("Test accuracy"); ax.set_ylim(0.96, 1.00); ax.grid(axis='y', alpha=0.3)
+    ax.set_title("Figure 4 — Image-level test accuracy across 20 seeds", fontsize=12)
+    plt.tight_layout()
+    out = os.path.join(PLOTS_DIR, "figure4", f"figure4{tag}.png")
+    os.makedirs(os.path.dirname(out), exist_ok=True)
+    plt.savefig(out, dpi=150); plt.close()
+    print(f"Saved → {out}")
+
+# Figure 6: image vs patient
+if args.only is None or args.only == "6":
+    fig, ax = plt.subplots(figsize=(10, 6))
+    for i, name in enumerate(PLOT_MODELS):
+        for pos, stype, color in [(i*2+0.7, "image", '#4C9BD4'),
+                                   (i*2+1.3, "patient", '#6DBF6D')]:
+            data = accs[name][stype]
+            bp = ax.boxplot(data, positions=[pos], widths=0.5,
+                            patch_artist=True, showfliers=True,
+                            flierprops=dict(marker='o', markersize=3))
+            bp['boxes'][0].set_facecolor(color)
+            ax.annotate(f"{np.median(data):.3f}", (pos, np.median(data)),
+                        fontsize=6, ha='center', va='bottom')
+    ax.legend(handles=[Patch(facecolor='#4C9BD4', label='Image-level split'),
+                       Patch(facecolor='#6DBF6D', label='Patient-level split')],
+              loc='lower right')
+    ax.set_xticks([p+1 for p in range(0, len(PLOT_MODELS)*2, 2)])
+    ax.set_xticklabels(PLOT_MODELS)
+    ax.set_ylabel("Test accuracy"); ax.set_ylim(0.70, 1.00); ax.grid(axis='y', alpha=0.3)
+    ax.set_title("Figure 6 — Image vs Patient-level test accuracy (20 seeds)", fontsize=13)
+    plt.tight_layout()
+    out = os.path.join(PLOTS_DIR, "figure6", f"figure6{tag}.png")
+    os.makedirs(os.path.dirname(out), exist_ok=True)
+    plt.savefig(out, dpi=150); plt.close()
+    print(f"Saved → {out}")
+
+print("DONE")

scripts/visualizations/figure5.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""figure5.py — PCA and t-SNE of VGG16 features, colored by class.
+
+Usage:
+    python scripts/visualizations/figure5.py
+    python scripts/visualizations/figure5.py --tag v2
+"""
+
+import os, sys, argparse
+import numpy as np
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from sklearn.decomposition import PCA
+from sklearn.manifold import TSNE
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__)))))
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__))))
+FEATURES_DIR = os.path.join(ROOT, "features")
+PLOTS_DIR = os.path.join(ROOT, "plots")
+SEED = 42
+CLASS_COLORS = {"Bengin cases": "#2CA02C", "Malignant cases": "#D62728",
+                "Normal cases": "#1F77B4"}
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--tag", default="", help="Append tag to filename")
+    args = ap.parse_args()
+    tag = f"_{args.tag}" if args.tag else ""
+
+    data = np.load(os.path.join(FEATURES_DIR, "VGG16_features.npz"),
+                   allow_pickle=True)
+    X, Y = data["X"], data["Y"]
+    pca50 = PCA(n_components=50, random_state=SEED).fit_transform(X)
+    pca2 = PCA(n_components=2, random_state=SEED).fit_transform(pca50)
+    tsne = TSNE(n_components=2, random_state=SEED, perplexity=30,
+                max_iter=1000).fit_transform(pca50)
+
+    fig, axes = plt.subplots(1, 2, figsize=(12, 5))
+    for ax, coords, title in [(axes[0], pca2, "(a) PCA"),
+                               (axes[1], tsne, "(b) t-SNE")]:
+        for cls in sorted(np.unique(Y)):
+            mask = Y == cls
+            label = cls.replace("Bengin cases", "Benign")
+            ax.scatter(coords[mask, 0], coords[mask, 1],
+                       c=CLASS_COLORS[cls], label=label,
+                       alpha=0.6, s=15, edgecolors="none")
+        ax.set_title(title, fontsize=12)
+        ax.set_xlabel("Component 1")
+        ax.set_ylabel("Component 2")
+        ax.legend(markerscale=2, fontsize=9)
+        ax.grid(alpha=0.2)
+
+    fig.suptitle("Figure 5 — PCA and t-SNE of VGG16 Features (by class)",
+                 fontsize=13)
+    plt.tight_layout()
+    out = os.path.join(PLOTS_DIR, "figure5", f"figure5{tag}.png")
+    os.makedirs(os.path.dirname(out), exist_ok=True)
+    plt.savefig(out, dpi=150)
+    plt.close()
+    print(f"Saved → {out}")
+
+
+if __name__ == "__main__":
+    main()

scripts/visualizations/figure_s2.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+"""figure_s2.py — PCA and t-SNE for all 5 CNN models, colored by class.
+Usage: python scripts/visualizations/figure_s2.py [--tag TAG]"""
+
+import os, sys, argparse
+import numpy as np
+import matplotlib; matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from sklearn.decomposition import PCA
+from sklearn.manifold import TSNE
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+FEATURES_DIR = os.path.join(ROOT, "features")
+PLOTS_DIR = os.path.join(ROOT, "plots")
+SEED = 42
+MODELS = ["VGG16", "MobileNetV2", "DenseNet121", "ResNet50", "EfficientNetB1"]
+CLASS_COLORS = {"Bengin cases": "#2CA02C", "Malignant cases": "#D62728", "Normal cases": "#1F77B4"}
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--tag", default="", help="Append tag to filename")
+args = ap.parse_args()
+tag = f"_{args.tag}" if args.tag else ""
+
+fig, axes = plt.subplots(2, 5, figsize=(22, 9))
+for mi, model_name in enumerate(MODELS):
+    data = np.load(os.path.join(FEATURES_DIR, f"{model_name}_features.npz"), allow_pickle=True)
+    X, Y = data["X"], data["Y"]
+    print(f"{model_name}: {X.shape}")
+    pca50 = PCA(n_components=50, random_state=SEED).fit_transform(X)
+    pca2 = PCA(n_components=2, random_state=SEED).fit_transform(pca50)
+    tsne = TSNE(n_components=2, random_state=SEED, perplexity=30, max_iter=800).fit_transform(pca50)
+    for row, coords, title in [(0, pca2, f"{model_name} PCA"), (1, tsne, f"{model_name} t-SNE")]:
+        ax = axes[row, mi]
+        for cls in sorted(np.unique(Y)):
+            mask = Y == cls; label = cls.replace("Bengin cases", "Benign")
+            ax.scatter(coords[mask,0], coords[mask,1], c=CLASS_COLORS[cls], label=label, alpha=0.5, s=8, edgecolors="none")
+        ax.set_title(title, fontsize=9)
+        ax.set_xlabel("Component 1" if row==0 else "t-SNE 1")
+        ax.set_ylabel("Component 2" if row==0 else "t-SNE 2")
+        ax.grid(alpha=0.2)
+
+handles = [plt.Line2D([0],[0], marker='o', color='w', markerfacecolor=c, markersize=8, label=l)
+           for l,c in zip(["Benign","Malignant","Normal"], ["#2CA02C","#D62728","#1F77B4"])]
+fig.legend(handles=handles, loc='lower center', ncol=3, fontsize=10, bbox_to_anchor=(0.5,-0.02))
+fig.suptitle("Figure S2 — PCA (top) and t-SNE (bottom) per model", fontsize=13, y=1.01)
+plt.tight_layout()
+out = os.path.join(PLOTS_DIR, "figure_s2", f"figure_s2{tag}.png")
+os.makedirs(os.path.dirname(out), exist_ok=True)
+plt.savefig(out, dpi=150, bbox_inches="tight")
+plt.close()
+print(f"Saved → {out}")

scripts/visualizations/figure_s3.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""figure_s3.py — Example CT images from estimated patient clusters.
+Usage: python scripts/visualizations/figure_s3.py [--tag TAG]"""
+
+import os, sys, csv, argparse
+import numpy as np
+import matplotlib; matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from PIL import Image
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+MANIFEST = os.path.join(ROOT, "results", "simple_patient_manifest.csv")
+DATASET = os.path.join(os.path.dirname(ROOT), "The IQ-OTHNCCD lung cancer dataset")
+PLOTS_DIR = os.path.join(ROOT, "plots")
+N_EXAMPLES = 5
+CLASS_MAP = {"Benign": ("Bengin cases", "B"), "Malignant": ("Malignant cases", "M"), "Normal": ("Normal cases", "N")}
+
+def f2n_back(fname):
+    prefix = fname[0]; num = int(fname.split("_")[1])
+    cls = {"B": ("Bengin cases", "Bengin"), "M": ("Malignant cases", "Malignant"), "N": ("Normal cases", "Normal")}[prefix]
+    return cls[0], f"{cls[1]} case ({num}).jpg"
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--tag", default="", help="Append tag to filename")
+args = ap.parse_args()
+tag = f"_{args.tag}" if args.tag else ""
+
+patients = {}
+with open(MANIFEST, newline="") as f:
+    reader = csv.DictReader(f)
+    img_col = "confirmed_images" if "confirmed_images" in reader.fieldnames else "images"
+    for row in reader:
+        imgs = row[img_col].split(";")
+        if imgs: patients[row["patient_id"]] = imgs
+
+fig, axes = plt.subplots(3, N_EXAMPLES, figsize=(12, 8))
+for row, (cls_label, (cls_dir, _)) in enumerate(CLASS_MAP.items()):
+    cls_patients = [(p, imgs) for p, imgs in patients.items() if p.lower().startswith(cls_label.lower()) and len(imgs) >= N_EXAMPLES]
+    if not cls_patients: continue
+    pid, imgs = cls_patients[0]
+    for col in range(N_EXAMPLES):
+        ax = axes[row, col]
+        try:
+            cls_dir_name, orig_fname = f2n_back(imgs[col])
+            img = Image.open(os.path.join(DATASET, cls_dir_name, orig_fname)).convert("L")
+            ax.imshow(img, cmap="gray")
+        except Exception as e:
+            ax.text(0.5, 0.5, f"error: {e}", ha="center", va="center", fontsize=7)
+        ax.set_xticks([]); ax.set_yticks([])
+        if col == 0: ax.set_ylabel(f"{cls_label}\nPatient {pid}", fontsize=9, rotation=0, labelpad=40, va="center")
+
+fig.suptitle("Figure S3 — Example CT images from estimated patient clusters", fontsize=12, y=1.01)
+plt.tight_layout()
+out = os.path.join(PLOTS_DIR, "figure_s3", f"figure_s3{tag}.png")
+os.makedirs(os.path.dirname(out), exist_ok=True)
+plt.savefig(out, dpi=150)
+plt.close()
+print(f"Saved → {out}")

scripts/visualizations/figure_s5.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""figure_s5.py — Confusion matrix for VGG16 patient-level classification.
+
+Reuses PatientLeakageClassifier.run(return_predictions=True) so the RF ranking,
+gamma grid, and final fit are not duplicated here — the same code path that
+produces the Figure 4/6 numbers also produces these predictions.
+
+Usage: python scripts/visualizations/figure_s5.py [--tag TAG]
+"""
+
+import os
+import sys
+import argparse
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
+
+os.environ.setdefault("OMP_NUM_THREADS", "1")
+os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
+os.environ.setdefault("MKL_NUM_THREADS", "1")
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__)))))
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+PLOTS_DIR = os.path.join(ROOT, "plots")
+
+from classes import PatientLeakageClassifier
+
+SEED = 20
+N_JOBS = 6
+CLASS_NAMES = ["Benign", "Malignant", "Normal"]
+
+
+def display_name(raw):
+    """Map a raw dataset label ('Bengin cases', ...) to a display class name."""
+    if raw.startswith("Bengin"):
+        return "Benign"
+    if raw.startswith("Malignant"):
+        return "Malignant"
+    return "Normal"
+
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--tag", default="")
+args = ap.parse_args()
+tag = f"_{args.tag}" if args.tag else ""
+
+clf = PatientLeakageClassifier(
+    os.path.join(ROOT, "results", "simple_patient_manifest.csv"),
+    os.path.join(ROOT, "features"), n_jobs=N_JOBS)
+r = clf.run("VGG16", SEED, "patient", return_predictions=True)
+print(f"Best: n={r['nfeat']}, gamma={r['gamma']:.6e}, "
+      f"CV={r['cv']:.4f}, Test={r['test']:.4f}")
+
+y_true = [display_name(c) for c in r["y_true"]]
+y_pred = [display_name(c) for c in r["y_pred"]]
+cm = confusion_matrix(y_true, y_pred, labels=CLASS_NAMES, normalize="true")
+
+fig, ax = plt.subplots(figsize=(6, 5))
+ConfusionMatrixDisplay(cm, display_labels=CLASS_NAMES).plot(
+    cmap="Blues", ax=ax, colorbar=True, values_format=".2f")
+ax.set_title("Figure S5 — Patient-level Confusion Matrix (VGG16)", fontsize=12)
+plt.tight_layout()
+out = os.path.join(PLOTS_DIR, "figure_s5", f"figure_s5{tag}.png")
+os.makedirs(os.path.dirname(out), exist_ok=True)
+plt.savefig(out, dpi=150)
+plt.close()
+print(f"Saved → {out}")

scripts/visualizations/manifest_html.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""
+manifest_html.py — Generate HTML pages for visually inspecting patient manifests.
+
+One HTML file per class, each patient's assigned images shown in a row.
+
+Usage:
+    python scripts/visualizations/manifest_html.py
+    python scripts/visualizations/manifest_html.py --method siamese
+    python scripts/visualizations/manifest_html.py --manifest results/simple_patient_manifest.csv --method pca50
+"""
+
+import os, sys, csv, argparse, base64
+from io import BytesIO
+from PIL import Image
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__)))))
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__))))
+
+DATASET = os.path.join(os.path.dirname(ROOT), "The IQ-OTHNCCD lung cancer dataset")
+RESULTS_DIR = os.path.join(ROOT, "results")
+HTML_DIR = os.path.join(ROOT, "plots", "html")
+
+MANIFESTS = {
+    "pca50":    os.path.join(RESULTS_DIR, "simple_patient_manifest.csv"),
+    "thumbnail": os.path.join(RESULTS_DIR, "thumbnail_patient_manifest.csv"),
+    "siamese":  os.path.join(RESULTS_DIR, "siamese_manifest.csv"),
+}
+
+THUMB_SIZE = 150  # px, display width
+
+
+def f2n_back(short_name):
+    """B_009 → ('Bengin cases', 'Bengin case (9).jpg')"""
+    prefix = short_name[0]
+    num = int(short_name.split("_")[1])
+    cls_map = {"B": ("Bengin cases", "Bengin"),
+               "M": ("Malignant cases", "Malignant"),
+               "N": ("Normal cases", "Normal")}
+    dir_name, file_prefix = cls_map[prefix]
+    return dir_name, f"{file_prefix} case ({num}).jpg"
+
+
+def img_to_b64(path, size=THUMB_SIZE):
+    """Load an image and return a base64 data URI."""
+    try:
+        img = Image.open(path).convert("L")
+        img.thumbnail((size, size), Image.LANCZOS)
+        buf = BytesIO()
+        img.save(buf, format="PNG")
+        return f"data:image/png;base64,{base64.b64encode(buf.getvalue()).decode()}"
+    except Exception:
+        return ""
+
+
+def build_html(manifest_path, method, class_name, class_dir, patients):
+    """Generate an HTML string for one class."""
+    rows = []
+    for pid, short_names in patients.items():
+        # Build image cards
+        cards = []
+        for sn in short_names:
+            dir_name, fname = f2n_back(sn)
+            img_path = os.path.join(DATASET, dir_name, fname)
+            b64 = img_to_b64(img_path)
+            if b64:
+                cards.append(
+                    f'<div class="card">'
+                    f'<img src="{b64}" alt="{sn}">'
+                    f'<div class="label">{sn}</div>'
+                    f'</div>')
+        if cards:
+            rows.append(
+                f'<div class="patient">'
+                f'<h3>{pid} <span class="count">({len(cards)} images)</span></h3>'
+                f'<div class="images">{"".join(cards)}</div>'
+                f'</div>')
+
+    return f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<title>{method} — {class_name}</title>
+<style>
+  body {{ font-family: -apple-system, sans-serif; background: #1a1a2e; color: #eee; margin: 20px; }}
+  h1 {{ color: #e94560; }}
+  .patient {{ margin-bottom: 30px; border-bottom: 1px solid #333; padding-bottom: 15px; }}
+  .patient h3 {{ margin: 0 0 8px 0; color: #0f3460; background: #16213e; padding: 6px 12px; border-radius: 4px; display: inline-block; }}
+  .count {{ font-weight: normal; color: #888; font-size: 0.85em; }}
+  .images {{ display: flex; flex-wrap: wrap; gap: 8px; }}
+  .card {{ background: #16213e; border-radius: 4px; overflow: hidden; width: {THUMB_SIZE + 20}px; }}
+  .card img {{ display: block; width: {THUMB_SIZE}px; height: {THUMB_SIZE}px; object-fit: contain; margin: 0 auto; background: #000; }}
+  .label {{ font-size: 9px; color: #aaa; text-align: center; padding: 4px; word-break: break-all; }}
+  a.nav {{ color: #e94560; margin-right: 15px; }}
+</style>
+</head>
+<body>
+<h1>{method} — {class_name} <small>({len(patients)} patients, {sum(len(v) for v in patients.values())} images)</small></h1>
+<p>
+  <a class="nav" href="{method}_Benign.html">Benign</a>
+  <a class="nav" href="{method}_Malignant.html">Malignant</a>
+  <a class="nav" href="{method}_Normal.html">Normal</a>
+</p>
+{"".join(rows)}
+</body>
+</html>"""
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--method", default=None,
+                    help="Which manifest method (pca50, thumbnail, siamese). "
+                         "Default: all.")
+    ap.add_argument("--manifest", default=None,
+                    help="Path to manifest CSV (overrides --method).")
+    args = ap.parse_args()
+
+    os.makedirs(HTML_DIR, exist_ok=True)
+
+    methods_to_run = [args.method] if args.method else list(MANIFESTS.keys())
+
+    for method in methods_to_run:
+        manifest_path = args.manifest or MANIFESTS.get(method)
+        if not manifest_path or not os.path.exists(manifest_path):
+            print(f"  {method}: manifest not found ({manifest_path})")
+            continue
+
+        # Load manifest, group by class
+        patients_by_class = {"Benign": {}, "Malignant": {}, "Normal": {}}
+        with open(manifest_path, newline="") as f:
+            reader = csv.DictReader(f)
+            img_col = ("images" if "images" in reader.fieldnames
+                       else "confirmed_images")
+            for row in reader:
+                cls = row.get("class", "").strip()
+                # Normalize class names
+                cls_lower = cls.lower()
+                if cls_lower in ("benign", "bengin"):
+                    cls = "Benign"
+                elif cls_lower in ("malignant", "malig"):
+                    cls = "Malignant"
+                elif cls_lower == "normal":
+                    cls = "Normal"
+                elif cls_lower == "unknown":
+                    # Infer from patient_id prefix
+                    pid = row.get("patient_id", "")
+                    if pid.lower().startswith("benign") or pid.lower().startswith("bengin"):
+                        cls = "Benign"
+                    elif pid.lower().startswith("malignant") or pid.lower().startswith("malig"):
+                        cls = "Malignant"
+                    elif pid.lower().startswith("normal"):
+                        cls = "Normal"
+                if cls not in patients_by_class:
+                    continue
+                pid = row["patient_id"]
+                imgs = row[img_col].split(";") if row[img_col] else []
+                if imgs:
+                    patients_by_class[cls][pid] = imgs
+
+        for cls_name in ["Benign", "Malignant", "Normal"]:
+            patients = patients_by_class[cls_name]
+            if not patients:
+                print(f"  {method}/{cls_name}: no patients, skipping")
+                continue
+
+            html = build_html(manifest_path, method, cls_name,
+                              "", patients)
+            out_path = os.path.join(HTML_DIR, f"{method}_{cls_name}.html")
+            with open(out_path, "w") as f:
+                f.write(html)
+            print(f"  Saved → plots/html/{method}_{cls_name}.html "
+                  f"({len(patients)} patients)")
+
+    print("DONE")
+
+
+if __name__ == "__main__":
+    main()

scripts/visualizations/manifest_tsne.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""manifest_tsne.py — visualize any patient-grouping manifest on the VGG16 t-SNE.
+
+Plots the same VGG16 feature t-SNE used elsewhere, but colored by the groups in
+a given manifest (siamese / pca50 / thumbnail). Because every manifest is drawn
+on the *identical* layout (same features, PCA, seed, perplexity), the resulting
+per-class figures are directly comparable across methods.
+
+For the siamese grouping this is diagnostic: if a siamese "patient" is a coherent
+patient it forms a tight island; if it's an over-merged chain, its color is
+smeared across feature space (VGG16 sees images the siamese wrongly linked).
+
+Usage:
+    conda activate fundus_imaging
+    python scripts/visualizations/manifest_tsne.py \
+        --manifest results/siamese_manifest.csv --name siamese
+    python scripts/visualizations/manifest_tsne.py \
+        --manifest results/simple_patient_manifest.csv --name pca50
+"""
+
+import os
+import sys
+import re
+import csv
+import argparse
+from collections import defaultdict
+
+import numpy as np
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from matplotlib.lines import Line2D
+from sklearn.decomposition import PCA
+from sklearn.manifold import TSNE
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__)))))
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+FEATURES_DIR = os.path.join(ROOT, "features")
+PLOTS_DIR = os.path.join(ROOT, "plots")
+
+RANDOM_STATE = 42
+CLASS_OF = {"B": "Benign", "M": "Malignant", "N": "Normal"}
+
+
+def f2n(fname):
+    """VGG16 feature filename -> manifest short name (e.g. 'B_001')."""
+    m = re.search(r"\((\d+)\)", fname)
+    num = int(m.group(1)) if m else None
+    for cls_key, prefix in [("Bengin cases", "B"), ("Malignant cases", "M"),
+                            ("Normal cases", "N")]:
+        if fname.startswith(cls_key.rstrip("s")):
+            return f"{prefix}_{num:03d}" if num else fname
+    return fname
+
+
+def load_manifest(path):
+    """Return {image_short_name: group_id}."""
+    mapping = {}
+    with open(path, newline="") as f:
+        for row in csv.DictReader(f):
+            for img in row["images"].split(";"):
+                if img:
+                    mapping[img] = row["patient_id"]
+    return mapping
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--manifest", required=True, help="Path to a grouping manifest CSV.")
+    ap.add_argument("--name", required=True, help="Short method name for titles/filenames.")
+    ap.add_argument("--features", default="VGG16", help="CNN feature set for the layout.")
+    ap.add_argument("--no-centroids", dest="centroids", action="store_false",
+                    help="Do not draw per-group centroid diamonds.")
+    ap.add_argument("--tag", default="")
+    args = ap.parse_args()
+    tag = f"_{args.tag}" if args.tag else ""
+
+    print(f"Loading {args.features} features ...")
+    data = np.load(os.path.join(FEATURES_DIR, f"{args.features}_features.npz"),
+                   allow_pickle=True)
+    X, filenames = data["X"], data["filenames"]
+    img_ids = np.array([f2n(f) for f in filenames])
+    img_class = np.array([CLASS_OF.get(i.split("_")[0], "?") for i in img_ids])
+
+    print("Computing PCA-50 + t-SNE (shared layout) ...")
+    n_pca = min(50, X.shape[0] - 1, X.shape[1])
+    X_pca = PCA(n_components=n_pca, random_state=RANDOM_STATE).fit_transform(X)
+    X_tsne = TSNE(n_components=2, perplexity=35, learning_rate="auto",
+                  init="pca", random_state=RANDOM_STATE).fit_transform(X_pca)
+
+    groups = load_manifest(args.manifest)
+    img_group = np.array([groups.get(i, "unassigned") for i in img_ids])
+
+    for cls in ["Benign", "Malignant", "Normal"]:
+        fig, ax = plt.subplots(figsize=(14, 10))
+        ax.scatter(X_tsne[:, 0], X_tsne[:, 1], c="lightgray", s=3, alpha=0.15)
+
+        mask = img_class == cls
+        class_groups = sorted(set(img_group[mask]))
+        n = len(class_groups)
+        # Order groups by size so the biggest (most likely over-merged) is obvious.
+        sizes = {g: int((img_group[mask] == g).sum()) for g in class_groups}
+        class_groups = sorted(class_groups, key=lambda g: -sizes[g])
+        cmap = plt.cm.tab20 if n <= 20 else plt.cm.gist_ncar
+
+        for gi, g in enumerate(class_groups):
+            color = cmap(gi % 20) if n <= 20 else cmap(gi / max(n - 1, 1))
+            gm = mask & (img_group == g)
+            ax.scatter(X_tsne[gm, 0], X_tsne[gm, 1], c=[color], s=18, alpha=0.8)
+            if args.centroids:
+                cx, cy = X_tsne[gm, 0].mean(), X_tsne[gm, 1].mean()
+                ax.scatter(cx, cy, c=[color], s=60, marker="D", edgecolors="black",
+                           linewidths=0.6, zorder=5)
+
+        biggest = class_groups[0]
+        legend_handles = [
+            Line2D([0], [0], marker="o", color="w", markerfacecolor="gray",
+                   markersize=8, label="Group images (dots)"),
+        ]
+        if args.centroids:
+            legend_handles.append(
+                Line2D([0], [0], marker="D", color="w", markerfacecolor="gray",
+                       markersize=8, label="Group centroids (diamonds)"))
+        ax.legend(handles=legend_handles, loc="lower right")
+        ax.set_title(f"{cls} — {args.name} groups on {args.features} t-SNE "
+                     f"({n} groups; largest={sizes[biggest]} imgs)")
+        ax.set_xlabel("t-SNE dim 1")
+        ax.set_ylabel("t-SNE dim 2")
+        plt.tight_layout()
+        out = os.path.join(PLOTS_DIR, "tsne", f"tsne_{args.name}_{cls}{tag}.png")
+        os.makedirs(os.path.dirname(out), exist_ok=True)
+        plt.savefig(out, dpi=150)
+        plt.close()
+        print(f"  Saved {out}")
+    print("DONE")
+
+
+if __name__ == "__main__":
+    main()

scripts/visualizations/siamese_similarity_tsne.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""siamese_similarity_tsne.py — t-SNE of the siamese's OWN similarity space.
+
+Unlike manifest_tsne.py (which recolors the VGG16 feature layout), this builds
+the layout directly from the siamese pairwise distance 1 - P(same-patient), so
+proximity reflects how the siamese model itself relates images. Points are
+colored by the siamese edge-rank groups.
+
+This is the diagnostic view for the over-merge: if the siamese collapses several
+patients together (over-confidence on IQ-OTH), the largest group forms one dense
+mass in its own space; coherent patients form tight, separated islands.
+
+Usage:
+    conda activate fundus_imaging
+    python scripts/visualizations/siamese_similarity_tsne.py
+"""
+
+import os
+import sys
+import re
+import csv
+import argparse
+import numpy as np
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from sklearn.manifold import TSNE
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__)))))
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+PLOTS_DIR = os.path.join(ROOT, "plots")
+DEFAULT_DATASET = os.path.join(os.path.dirname(ROOT),
+                               "The IQ-OTHNCCD lung cancer dataset")
+
+from classes import SiamesePatientMatcher
+
+CLASS_DIRS = {"Bengin cases": "Benign", "Malignant cases": "Malignant",
+              "Normal cases": "Normal"}
+VALID_EXT = (".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp")
+RANDOM_STATE = 42
+
+
+def f2n(fname):
+    m = re.search(r"\((\d+)\)", fname)
+    num = int(m.group(1)) if m else None
+    for cls_key, prefix in [("Bengin cases", "B"), ("Malignant cases", "M"),
+                            ("Normal cases", "N")]:
+        if fname.startswith(cls_key.rstrip("s")):
+            return f"{prefix}_{num:03d}" if num else fname
+    return fname
+
+
+def load_manifest(path):
+    mapping = {}
+    with open(path, newline="") as f:
+        for row in csv.DictReader(f):
+            for img in row["images"].split(";"):
+                if img:
+                    mapping[img] = row["patient_id"]
+    return mapping
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--model", default=os.path.join(ROOT, "models", "siamese_resnet18.pt"))
+    ap.add_argument("--backbone", default="resnet18")
+    ap.add_argument("--dataset", default=DEFAULT_DATASET)
+    ap.add_argument("--manifest", default=os.path.join(ROOT, "results", "siamese_manifest.csv"))
+    ap.add_argument("--name", default="siamese_sim")
+    ap.add_argument("--highlight-largest", type=int, default=0,
+                    help="Grey all points and bold only the N largest groups.")
+    ap.add_argument("--tag", default="")
+    args = ap.parse_args()
+    tag = f"_{args.tag}" if args.tag else ""
+
+    matcher = SiamesePatientMatcher(args.model, backbone=args.backbone, input_size=224)
+    groups = load_manifest(args.manifest)
+
+    for class_dir, cls in CLASS_DIRS.items():
+        cpath = os.path.join(args.dataset, class_dir)
+        files = sorted(f for f in os.listdir(cpath) if f.lower().endswith(VALID_EXT))
+        paths = [os.path.join(cpath, f) for f in files]
+        ids = [f2n(f) for f in files]
+        print(f"\n{cls}: {len(paths)} images")
+
+        # Siamese pairwise distance -> t-SNE on precomputed distances.
+        emb = matcher.embed_images(paths)
+        P = matcher._dense_prob_matrix(emb)
+        dist = np.clip(1.0 - P, 0.0, None)
+        np.fill_diagonal(dist, 0.0)
+        perp = max(5, min(30, (len(paths) - 1) // 3))
+        X = TSNE(n_components=2, metric="precomputed", init="random",
+                 perplexity=perp, random_state=RANDOM_STATE).fit_transform(dist)
+
+        img_group = np.array([groups.get(i, "unassigned") for i in ids])
+        g_order = sorted(set(img_group), key=lambda g: -(img_group == g).sum())
+        n = len(g_order)
+
+        fig, ax = plt.subplots(figsize=(14, 10))
+        if args.highlight_largest > 0:
+            # Grey everything, then bold only the N largest groups.
+            ax.scatter(X[:, 0], X[:, 1], c="lightgray", s=12, alpha=0.5)
+            hl = g_order[:args.highlight_largest]
+            hl_cmap = plt.cm.tab10
+            for gi, g in enumerate(hl):
+                gm = img_group == g
+                ax.scatter(X[gm, 0], X[gm, 1], c=[hl_cmap(gi)], s=28, alpha=0.9,
+                           edgecolors="black", linewidths=0.3,
+                           label=f"{g} ({gm.sum()} imgs)")
+            ax.legend(loc="lower right", title="Largest siamese groups")
+            ax.set_title(f"{cls} — siamese similarity t-SNE "
+                         f"(largest {len(hl)} of {n} groups highlighted)")
+        else:
+            cmap = plt.cm.tab20 if n <= 20 else plt.cm.gist_ncar
+            for gi, g in enumerate(g_order):
+                color = cmap(gi % 20) if n <= 20 else cmap(gi / max(n - 1, 1))
+                gm = img_group == g
+                ax.scatter(X[gm, 0], X[gm, 1], c=[color], s=18, alpha=0.8)
+            biggest = (img_group == g_order[0]).sum()
+            ax.set_title(f"{cls} — siamese similarity t-SNE "
+                         f"({n} groups; largest={biggest} imgs)")
+        ax.set_xlabel("t-SNE dim 1 (siamese distance)")
+        ax.set_ylabel("t-SNE dim 2 (siamese distance)")
+        plt.tight_layout()
+        suffix = "_highlight" if args.highlight_largest > 0 else ""
+        out = os.path.join(PLOTS_DIR, "tsne", f"tsne_{args.name}_{cls}{suffix}{tag}.png")
+        os.makedirs(os.path.dirname(out), exist_ok=True)
+        plt.savefig(out, dpi=150)
+        plt.close()
+        print(f"  Saved {out}")
+    print("DONE")
+
+
+if __name__ == "__main__":
+    main()

scripts/visualizations/simple_patient_tsne.py

@@ -0,0 +1,200 @@
+
+
+
+import os, sys, re, csv, json
+from collections import defaultdict
+import numpy as np
+import matplotlib
+import matplotlib.pyplot as plt
+from sklearn.decomposition import PCA
+from sklearn.manifold import TSNE
+from sklearn.cluster import KMeans
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+import argparse
+#!/usr/bin/env python3
+"""
+simple_patient_tsne.py
+The short path:
+  1. Load VGG16 features
+  2. K-means per class (15/40/55 patients) in PCA-50d space
+  3. Optional: iterative centroid refinement
+  4. Plot t-SNE colored by cluster, with centroid labels
+Usage:
+    conda activate fundus_imaging
+    python scripts/simple_patient_tsne.py
+"""
+matplotlib.use("Agg")
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+FEATURES_DIR = os.path.join(PROJECT_ROOT, "features")
+PLOTS_DIR = os.path.join(PROJECT_ROOT, "plots")
+RESULTS_DIR = os.path.join(PROJECT_ROOT, "results")
+os.makedirs(PLOTS_DIR, exist_ok=True)
+os.makedirs(RESULTS_DIR, exist_ok=True)
+PATIENT_COUNTS = {"Bengin cases": 15, "Malignant cases": 40, "Normal cases": 55}
+CLASS_NAMES = {"Bengin cases": "Benign", "Malignant cases": "Malignant", "Normal cases": "Normal"}
+RANDOM_STATE = 42
+# ---------------------------------------------------------------------------
+# 1. Load VGG16 features
+# ---------------------------------------------------------------------------
+print("Loading VGG16 features ...")
+data = np.load(os.path.join(FEATURES_DIR, "VGG16_features.npz"), allow_pickle=True)
+X, Y, filenames = data["X"], data["Y"], data["filenames"]
+def f2n(fname):
+    m = re.search(r'\((\d+)\)', fname)
+    num = int(m.group(1)) if m else None
+    for cls_key, prefix in [("Bengin cases","B"),("Malignant cases","M"),("Normal cases","N")]:
+        if fname.startswith(cls_key.rstrip("s")):
+            return f"{prefix}_{num:03d}" if num else fname
+    return fname
+img_nums = np.array([f2n(f) for f in filenames])
+# ---------------------------------------------------------------------------
+# 2. K-means per class in PCA-50d space
+# ---------------------------------------------------------------------------
+print("Clustering in PCA-50d space ...")
+n_pca = min(50, X.shape[0] - 1, X.shape[1])
+X_pca = PCA(n_components=n_pca, random_state=RANDOM_STATE).fit_transform(X)
+image_to_patient = {}
+patient_to_images = defaultdict(list)
+for class_name, k in PATIENT_COUNTS.items():
+    mask = Y == class_name
+    X_class = X_pca[mask]
+    idx_class = np.where(mask)[0]
+    kmeans = KMeans(n_clusters=k, random_state=RANDOM_STATE, n_init=20)
+    labels = kmeans.fit_predict(X_class)
+    prefix = {"Bengin cases": "Benign", "Malignant cases": "Malignant", "Normal cases": "Normal"}[class_name]
+    for i, cluster_id in enumerate(labels):
+        pid = f"{prefix}_{cluster_id:02d}"
+        img = img_nums[idx_class[i]]
+        image_to_patient[img] = pid
+        patient_to_images[pid].append(img)
+print(f"  {len(patient_to_images)} patients, {len(image_to_patient)} images")
+# ---------------------------------------------------------------------------
+# 3. Iterative centroid refinement (optional, 3 passes)
+# ---------------------------------------------------------------------------
+print("Refining assignments (nearest-centroid, 5 passes) ...")
+for iteration in range(5):
+    # Compute centroids
+    centroids = {}
+    for pid, imgs in patient_to_images.items():
+        idxs = [np.where(img_nums == img)[0][0] for img in imgs]
+        centroids[pid] = X_pca[idxs].mean(axis=0)
+    # Reassign
+    moves = 0
+    for class_name in PATIENT_COUNTS:
+        mask = Y == class_name
+        for i in np.where(mask)[0]:
+            img = img_nums[i]
+            old_pid = image_to_patient[img]
+            # Find nearest centroid in same class
+            best_pid = old_pid
+            best_dist = float('inf')
+            for pid, c in centroids.items():
+                if pid.startswith(CLASS_NAMES[class_name]):
+                    d = float(np.linalg.norm(X_pca[i] - c))
+                    if d < best_dist:
+                        best_dist = d
+                        best_pid = pid
+            if best_pid != old_pid:
+                patient_to_images[old_pid].remove(img)
+                patient_to_images[best_pid].append(img)
+                image_to_patient[img] = best_pid
+                moves += 1
+    print(f"  Pass {iteration+1}: {moves} moves")
+    if moves == 0:
+        break
+# ---------------------------------------------------------------------------
+# 4. t-SNE
+# ---------------------------------------------------------------------------
+print("Computing t-SNE ...")
+X_tsne = TSNE(n_components=2, perplexity=35, learning_rate="auto",
+              init="pca", random_state=RANDOM_STATE).fit_transform(X_pca)
+# ---------------------------------------------------------------------------
+# 5. Plot — one figure per class
+# ---------------------------------------------------------------------------
+for class_name, display_name in CLASS_NAMES.items():
+    fig, ax = plt.subplots(1, 1, figsize=(14, 10))
+    ax.scatter(X_tsne[:, 0], X_tsne[:, 1], c="lightgray", s=3, alpha=0.15)
+    mask = Y == class_name
+    class_pids = sorted([p for p in patient_to_images if p.startswith(display_name)])
+    n_patients = len(class_pids)
+    cmap = plt.cm.tab20 if n_patients <= 20 else plt.cm.gist_ncar
+    for pi, pid in enumerate(class_pids):
+        color = cmap(pi % 20) if n_patients <= 20 else cmap(pi / max(n_patients-1, 1))
+        pts_x, pts_y = [], []
+        for img in patient_to_images[pid]:
+            i = np.where(img_nums == img)[0][0]
+            pts_x.append(X_tsne[i, 0])
+            pts_y.append(X_tsne[i, 1])
+        ax.scatter(pts_x, pts_y, c=[color], s=18, alpha=0.8, label='_nolegend_')
+        # Centroid diamond (no label)
+        cx, cy = np.mean(pts_x), np.mean(pts_y)
+        ax.scatter(cx, cy, c=[color], s=60, marker='D', edgecolors='black',
+                  linewidths=0.6, zorder=5, label='_nolegend_')
+    # Legend elements
+    from matplotlib.lines import Line2D
+    legend_elements = [
+        Line2D([0], [0], marker='o', color='w', markerfacecolor='gray',
+               markersize=8, label='Patient images (dots)'),
+        Line2D([0], [0], marker='D', color='w', markerfacecolor='gray',
+               markersize=8, label='Patient centroids (diamonds)'),
+    ]
+    ax.legend(handles=legend_elements, loc='lower right')
+    ax.set_title(f"{display_name} — VGG16 t-SNE ({n_patients} patients)")
+    ax.set_xlabel("t-SNE dim 1")
+    ax.set_ylabel("t-SNE dim 2")
+    plt.tight_layout()
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--tag", default="", help="Append tag to filename")
+    args = ap.parse_args()
+    tag = f"_{args.tag}" if args.tag else ""
+
+    out = os.path.join(PLOTS_DIR, "tsne", f"tsne_{display_name}{tag}.png")
+    os.makedirs(os.path.dirname(out), exist_ok=True)
+    plt.savefig(out, dpi=150)
+    plt.close()
+    print(f"  Saved {out}")
+# ---------------------------------------------------------------------------
+# 6. Save assignments
+# ---------------------------------------------------------------------------
+manifest = []
+for pid in sorted(patient_to_images.keys()):
+    imgs = sorted(patient_to_images[pid])
+    manifest.append({"patient_id": pid, "class": pid.split("_")[0],
+                     "n_images": len(imgs), "images": ";".join(imgs)})
+with open(os.path.join(RESULTS_DIR, "simple_patient_manifest.csv"), "w", newline="") as f:
+    w = csv.DictWriter(f, fieldnames=["patient_id", "class", "n_images", "images"])
+    w.writeheader()
+    w.writerows(manifest)
+print(f"\nSaved simple_patient_manifest.csv ({len(manifest)} patients, "
+      f"{sum(m['n_images'] for m in manifest)} images)")
+print("DONE")