2026001
This commit is contained in:
rpotter6298
@@ -0,0 +1,245 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
compare_clustering_methods.py
|
||||
|
||||
Compare two patient-clustering approaches against the v8 "reference":
|
||||
1. Thumbnail-based (64×64 grayscale K-means — the manuscript's method)
|
||||
2. Feature-based (PCA-50d of VGG16 features → K-means — our method)
|
||||
|
||||
Metrics (all label-invariant):
|
||||
- Adjusted Rand Index (ARI)
|
||||
- Normalized Mutual Information (NMI)
|
||||
- V-measure (homogeneity + completeness)
|
||||
|
||||
Usage:
|
||||
conda activate fundus_imaging
|
||||
python scripts/compare_clustering_methods.py
|
||||
"""
|
||||
|
||||
import os, sys, csv, re
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from sklearn.metrics import (
|
||||
adjusted_rand_score,
|
||||
normalized_mutual_info_score,
|
||||
homogeneity_completeness_v_measure,
|
||||
)
|
||||
from sklearn.decomposition import PCA
|
||||
from sklearn.cluster import KMeans
|
||||
from PIL import Image
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DATASET_PATH = os.path.expanduser("~/Documents/data_leakage/The IQ-OTHNCCD lung cancer dataset")
|
||||
MANIFEST_V8 = os.path.join(ROOT, ".archive", "results", "patient_manifest_v8.csv")
|
||||
MANIFEST_SIMPLE = os.path.join(ROOT, "results", "simple_patient_manifest.csv")
|
||||
FEATURES_PATH = os.path.join(ROOT, "features", "VGG16_features.npz")
|
||||
|
||||
PATIENT_COUNTS = {
|
||||
"Bengin cases": 15,
|
||||
"Malignant cases": 40,
|
||||
"Normal cases": 55,
|
||||
}
|
||||
|
||||
CLASS_ORDER = ["Bengin cases", "Malignant cases", "Normal cases"]
|
||||
|
||||
SEED = 42
|
||||
THUMBNAIL_SIZE = (64, 64)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def f2n(fname):
|
||||
m = re.search(r"\((\d+)\)", fname)
|
||||
num = int(m.group(1)) if m else None
|
||||
for cls_key, prefix in [
|
||||
("Bengin cases", "B"),
|
||||
("Malignant cases", "M"),
|
||||
("Normal cases", "N"),
|
||||
]:
|
||||
if fname.startswith(cls_key.rstrip("s")):
|
||||
return f"{prefix}_{num:03d}" if num else fname
|
||||
return fname
|
||||
|
||||
|
||||
def load_v8_assignments(path):
|
||||
"""Load v8 manifest, return {image_name: patient_id}."""
|
||||
mapping = {}
|
||||
with open(path, newline="") as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
for img in row["confirmed_images"].split(";"):
|
||||
if img:
|
||||
mapping[img] = row["patient_id"]
|
||||
return mapping
|
||||
|
||||
|
||||
def load_simple_assignments(path):
|
||||
"""Load simple feature-based manifest, return {image_name: patient_id}."""
|
||||
mapping = {}
|
||||
with open(path, newline="") as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
for img in row["images"].split(";"):
|
||||
if img:
|
||||
mapping[img] = row["patient_id"]
|
||||
return mapping
|
||||
|
||||
|
||||
def run_thumbnail_clustering(dataset_path, filenames, labels):
|
||||
"""Run 64×64 grayscale thumbnail K-means (manuscript method)."""
|
||||
groups = np.array([None] * len(labels), dtype=object)
|
||||
|
||||
for class_name, n_clusters in PATIENT_COUNTS.items():
|
||||
idx_class = np.where(labels == class_name)[0]
|
||||
|
||||
# Load thumbnails
|
||||
X = []
|
||||
for i in idx_class:
|
||||
fname = filenames[i]
|
||||
img_path = os.path.join(dataset_path, class_name, fname)
|
||||
try:
|
||||
img = Image.open(img_path).convert("L")
|
||||
img = img.resize(THUMBNAIL_SIZE)
|
||||
img_arr = np.array(img, dtype=np.float32) / 255.0
|
||||
X.append(img_arr.flatten())
|
||||
except Exception:
|
||||
X.append(np.zeros(64 * 64, dtype=np.float32))
|
||||
X = np.array(X, dtype=np.float32)
|
||||
|
||||
# PCA → 50d
|
||||
n_pca = min(50, X.shape[0] - 1, X.shape[1])
|
||||
pca = PCA(n_components=n_pca, random_state=SEED)
|
||||
X_pca = pca.fit_transform(X)
|
||||
|
||||
# K-means
|
||||
kmeans = KMeans(n_clusters=n_clusters, random_state=SEED, n_init=20)
|
||||
clusters = kmeans.fit_predict(X_pca)
|
||||
|
||||
for i, cluster_id in zip(idx_class, clusters):
|
||||
groups[i] = f"{class_name}_cluster_{cluster_id}"
|
||||
|
||||
sizes = np.bincount(clusters)
|
||||
print(f" {class_name}: k={n_clusters}, sizes min={sizes.min()} max={sizes.max()} mean={sizes.mean():.1f}")
|
||||
|
||||
return groups
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
print("Loading VGG16 features for filename/label reference...")
|
||||
data = np.load(FEATURES_PATH, allow_pickle=True)
|
||||
all_filenames = data["filenames"]
|
||||
all_labels = data["Y"]
|
||||
X_vgg16 = data["X"]
|
||||
|
||||
print(f" {len(all_filenames)} images across {len(np.unique(all_labels))} classes")
|
||||
|
||||
# Load reference (v8) and our feature-based assignments
|
||||
v8_map = load_v8_assignments(MANIFEST_V8)
|
||||
simple_map = load_simple_assignments(MANIFEST_SIMPLE)
|
||||
|
||||
# Build per-image label arrays for all three methods, aligned by filename
|
||||
# We need all images that exist in ALL three
|
||||
v8_labels_list = []
|
||||
simple_labels_list = []
|
||||
thumb_labels_list = [] # filled after clustering
|
||||
common_filenames = []
|
||||
common_labels = []
|
||||
|
||||
# First, run thumbnail clustering
|
||||
print("\nRunning thumbnail-based K-means (manuscript method)...")
|
||||
thumb_groups = run_thumbnail_clustering(DATASET_PATH, all_filenames, all_labels)
|
||||
|
||||
# Build thumbnail mapping (using short names like B_001)
|
||||
thumb_map = {}
|
||||
for fname, group in zip(all_filenames, thumb_groups):
|
||||
short = f2n(fname)
|
||||
for cls in CLASS_ORDER:
|
||||
if group.startswith(cls):
|
||||
cluster_id = int(group.split("_cluster_")[-1])
|
||||
prefix = {"Bengin cases": "Benign", "Malignant cases": "Malignant", "Normal cases": "Normal"}[cls]
|
||||
thumb_map[short] = f"thumb_{prefix}_{cluster_id:02d}"
|
||||
break
|
||||
|
||||
# Filter to images present in all three
|
||||
# Feature filenames are like "Bengin case (1).jpg", manifests use "B_001"
|
||||
for fname, label in zip(all_filenames, all_labels):
|
||||
short_name = f2n(fname)
|
||||
v8_id = v8_map.get(short_name)
|
||||
simple_id = simple_map.get(short_name)
|
||||
thumb_id = thumb_map.get(short_name)
|
||||
if v8_id and simple_id and thumb_id:
|
||||
common_filenames.append(short_name)
|
||||
common_labels.append(label)
|
||||
v8_labels_list.append(v8_id)
|
||||
simple_labels_list.append(simple_id)
|
||||
thumb_labels_list.append(thumb_id)
|
||||
|
||||
print(f"\nImages common to all three methods: {len(common_filenames)}")
|
||||
|
||||
# Convert to numpy arrays
|
||||
v8_labels_arr = np.array(v8_labels_list)
|
||||
simple_labels_arr = np.array(simple_labels_list)
|
||||
thumb_labels_arr = np.array(thumb_labels_list)
|
||||
common_labels_arr = np.array(common_labels)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Compute agreement metrics — per-class and overall
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_metrics(ref, pred, name):
|
||||
"""Compute clustering agreement metrics against reference."""
|
||||
ari = adjusted_rand_score(ref, pred)
|
||||
nmi = normalized_mutual_info_score(ref, pred)
|
||||
h, c, v = homogeneity_completeness_v_measure(ref, pred)
|
||||
return {"name": name, "ARI": ari, "NMI": nmi, "Homogeneity": h, "Completeness": c, "V_measure": v}
|
||||
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("OVERALL AGREEMENT WITH v8 REFERENCE")
|
||||
print("=" * 80)
|
||||
|
||||
results = []
|
||||
for name, pred in [("Thumbnail (manuscript)", thumb_labels_arr), ("Feature-based (ours)", simple_labels_arr)]:
|
||||
r = compute_metrics(v8_labels_arr, pred, name)
|
||||
results.append(r)
|
||||
|
||||
print(f"\n{'Method':<30s} {'ARI':>8s} {'NMI':>8s} {'Homog':>8s} {'Compl':>8s} {'V_meas':>8s}")
|
||||
print("-" * 72)
|
||||
for r in results:
|
||||
print(f"{r['name']:<30s} {r['ARI']:>8.4f} {r['NMI']:>8.4f} {r['Homogeneity']:>8.4f} {r['Completeness']:>8.4f} {r['V_measure']:>8.4f}")
|
||||
|
||||
# Per-class breakdown
|
||||
print("\n" + "=" * 80)
|
||||
print("PER-CLASS ARI WITH v8 REFERENCE")
|
||||
print("=" * 80)
|
||||
print(f"\n{'Class':<20s} {'Thumbnail':>10s} {'Feature-based':>15s}")
|
||||
print("-" * 47)
|
||||
for cls in CLASS_ORDER:
|
||||
mask = common_labels_arr == cls
|
||||
if mask.sum() < 2:
|
||||
continue
|
||||
thumb_ari = adjusted_rand_score(v8_labels_arr[mask], thumb_labels_arr[mask])
|
||||
feat_ari = adjusted_rand_score(v8_labels_arr[mask], simple_labels_arr[mask])
|
||||
better = " ←" if feat_ari > thumb_ari else ""
|
||||
print(f"{cls:<20s} {thumb_ari:>10.4f} {feat_ari:>15.4f}{better}")
|
||||
|
||||
# Also: direct agreement between thumbnail and feature-based
|
||||
print("\n" + "=" * 80)
|
||||
print("THUMBNAIL vs FEATURE-BASED (direct agreement)")
|
||||
print("=" * 80)
|
||||
direct = compute_metrics(thumb_labels_arr, simple_labels_arr, "Thumb vs Feature")
|
||||
print(f" ARI={direct['ARI']:.4f} NMI={direct['NMI']:.4f} V_measure={direct['V_measure']:.4f}")
|
||||
|
||||
print("\nDONE")
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""compare_patient_groupings.py
|
||||
|
||||
Compare the three estimated patient groupings of the IQ-OTH/NCCD
|
||||
("lung_effnet") dataset against each other:
|
||||
|
||||
siamese -> results/siamese_manifest.csv
|
||||
pca50 -> results/simple_patient_manifest.csv (PCA-50 CNN-feature K-means)
|
||||
thumbnail -> results/thumbnail_patient_manifest.csv (64x64 grayscale K-means)
|
||||
|
||||
Downstream classification accuracy is affected similarly by all three, so the
|
||||
question this answers is: do the three methods actually partition the images
|
||||
differently? If they agree closely, the method choice is cosmetic; if they
|
||||
diverge, the groupings are genuinely different partitions — which, combined with
|
||||
the Task06 ground-truth validation (where siamese scored higher purity/capture),
|
||||
is what makes the siamese work worthwhile.
|
||||
|
||||
Metrics are all label-invariant (cluster-id names don't matter):
|
||||
ARI - Adjusted Rand Index (chance-corrected pair agreement)
|
||||
NMI - Normalized Mutual Information
|
||||
V - V-measure (harmonic mean of homogeneity & completeness)
|
||||
|
||||
Outputs:
|
||||
results/compare_patient_groupings.json
|
||||
plots/analysis/grouping_agreement_heatmap.png
|
||||
|
||||
Usage:
|
||||
conda activate fundus_imaging
|
||||
python scripts/analysis/compare_patient_groupings.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import csv
|
||||
import json
|
||||
from itertools import combinations
|
||||
|
||||
import numpy as np
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.metrics import (
|
||||
adjusted_rand_score,
|
||||
normalized_mutual_info_score,
|
||||
homogeneity_completeness_v_measure,
|
||||
)
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
RESULTS_DIR = os.path.join(ROOT, "results")
|
||||
PLOTS_DIR = os.path.join(ROOT, "plots", "analysis")
|
||||
|
||||
METHODS = {
|
||||
"siamese": "siamese_manifest.csv",
|
||||
"pca50": "simple_patient_manifest.csv",
|
||||
"thumbnail": "thumbnail_patient_manifest.csv",
|
||||
}
|
||||
METHOD_ORDER = ["siamese", "pca50", "thumbnail"]
|
||||
CLASS_OF = {"B": "Benign", "M": "Malignant", "N": "Normal"}
|
||||
|
||||
|
||||
def load_manifest(path):
|
||||
"""Return {image_short_name: group_id} from a manifest CSV."""
|
||||
mapping = {}
|
||||
with open(path, newline="") as f:
|
||||
for row in csv.DictReader(f):
|
||||
for img in row["images"].split(";"):
|
||||
if img:
|
||||
mapping[img] = row["patient_id"]
|
||||
return mapping
|
||||
|
||||
|
||||
def size_stats(labels):
|
||||
"""Group-size distribution for one method's label array."""
|
||||
_, counts = np.unique(labels, return_counts=True)
|
||||
return {
|
||||
"n_groups": int(len(counts)),
|
||||
"min": int(counts.min()),
|
||||
"median": float(np.median(counts)),
|
||||
"mean": float(counts.mean()),
|
||||
"max": int(counts.max()),
|
||||
"singletons": int((counts == 1).sum()),
|
||||
}
|
||||
|
||||
|
||||
def agreement(a, b):
|
||||
h, c, v = homogeneity_completeness_v_measure(a, b)
|
||||
return {"ARI": float(adjusted_rand_score(a, b)),
|
||||
"NMI": float(normalized_mutual_info_score(a, b)),
|
||||
"V": float(v)}
|
||||
|
||||
|
||||
def main():
|
||||
maps = {m: load_manifest(os.path.join(RESULTS_DIR, f))
|
||||
for m, f in METHODS.items()}
|
||||
|
||||
# Align on images present in all three (should be the full 1,097).
|
||||
common = sorted(set.intersection(*[set(mp) for mp in maps.values()]))
|
||||
classes = np.array([CLASS_OF.get(img.split("_")[0], "?") for img in common])
|
||||
labels = {m: np.array([maps[m][img] for img in common]) for m in METHOD_ORDER}
|
||||
print(f"Aligned on {len(common)} images common to all three methods.\n")
|
||||
|
||||
# --- Structural summary --------------------------------------------------
|
||||
print("=" * 72)
|
||||
print("GROUP STRUCTURE PER METHOD")
|
||||
print("=" * 72)
|
||||
print(f"{'method':<12s} {'groups':>7s} {'min':>5s} {'median':>7s} "
|
||||
f"{'mean':>6s} {'max':>5s} {'singletons':>11s}")
|
||||
structure = {}
|
||||
for m in METHOD_ORDER:
|
||||
s = size_stats(labels[m])
|
||||
structure[m] = s
|
||||
print(f"{m:<12s} {s['n_groups']:>7d} {s['min']:>5d} {s['median']:>7.1f} "
|
||||
f"{s['mean']:>6.1f} {s['max']:>5d} {s['singletons']:>11d}")
|
||||
|
||||
# --- Pairwise agreement --------------------------------------------------
|
||||
print("\n" + "=" * 72)
|
||||
print("PAIRWISE AGREEMENT (how similarly the methods partition the images)")
|
||||
print("=" * 72)
|
||||
print(f"{'pair':<24s} {'ARI':>8s} {'NMI':>8s} {'V':>8s}")
|
||||
print("-" * 52)
|
||||
pairwise = {}
|
||||
ari_matrix = np.eye(len(METHOD_ORDER))
|
||||
for i, j in combinations(range(len(METHOD_ORDER)), 2):
|
||||
a, b = METHOD_ORDER[i], METHOD_ORDER[j]
|
||||
g = agreement(labels[a], labels[b])
|
||||
pairwise[f"{a}_vs_{b}"] = g
|
||||
ari_matrix[i, j] = ari_matrix[j, i] = g["ARI"]
|
||||
print(f"{a+' vs '+b:<24s} {g['ARI']:>8.3f} {g['NMI']:>8.3f} {g['V']:>8.3f}")
|
||||
|
||||
# --- Per-class ARI -------------------------------------------------------
|
||||
print("\n" + "=" * 72)
|
||||
print("PER-CLASS ARI (agreement within each diagnostic class)")
|
||||
print("=" * 72)
|
||||
print(f"{'pair':<24s} " + " ".join(f"{c:>10s}" for c in ["Benign", "Malignant", "Normal"]))
|
||||
print("-" * 60)
|
||||
per_class = {}
|
||||
for i, j in combinations(range(len(METHOD_ORDER)), 2):
|
||||
a, b = METHOD_ORDER[i], METHOD_ORDER[j]
|
||||
row = {}
|
||||
cells = []
|
||||
for c in ["Benign", "Malignant", "Normal"]:
|
||||
mask = classes == c
|
||||
ari = float(adjusted_rand_score(labels[a][mask], labels[b][mask]))
|
||||
row[c] = ari
|
||||
cells.append(f"{ari:>10.3f}")
|
||||
per_class[f"{a}_vs_{b}"] = row
|
||||
print(f"{a+' vs '+b:<24s} " + " ".join(cells))
|
||||
|
||||
# --- Odd-one-out: mean ARI of each method vs the other two ---------------
|
||||
print("\n" + "=" * 72)
|
||||
print("MEAN ARI OF EACH METHOD VS THE OTHER TWO (lower = most distinct)")
|
||||
print("=" * 72)
|
||||
mean_ari = {}
|
||||
for i, m in enumerate(METHOD_ORDER):
|
||||
others = [ari_matrix[i, j] for j in range(len(METHOD_ORDER)) if j != i]
|
||||
mean_ari[m] = float(np.mean(others))
|
||||
print(f" {m:<12s} {mean_ari[m]:.3f}")
|
||||
odd = min(mean_ari, key=mean_ari.get)
|
||||
print(f"\n Most distinct grouping: {odd}")
|
||||
|
||||
# --- Save + heatmap ------------------------------------------------------
|
||||
out = {
|
||||
"n_images": len(common),
|
||||
"structure": structure,
|
||||
"pairwise": pairwise,
|
||||
"per_class_ARI": per_class,
|
||||
"mean_ari_vs_others": mean_ari,
|
||||
"most_distinct": odd,
|
||||
}
|
||||
os.makedirs(RESULTS_DIR, exist_ok=True)
|
||||
out_path = os.path.join(RESULTS_DIR, "compare_patient_groupings.json")
|
||||
with open(out_path, "w") as f:
|
||||
json.dump(out, f, indent=2)
|
||||
print(f"\nSaved metrics → {out_path}")
|
||||
|
||||
fig, ax = plt.subplots(figsize=(5.5, 4.5))
|
||||
im = ax.imshow(ari_matrix, vmin=0, vmax=1, cmap="viridis")
|
||||
ax.set_xticks(range(len(METHOD_ORDER)))
|
||||
ax.set_yticks(range(len(METHOD_ORDER)))
|
||||
ax.set_xticklabels(METHOD_ORDER)
|
||||
ax.set_yticklabels(METHOD_ORDER)
|
||||
for i in range(len(METHOD_ORDER)):
|
||||
for j in range(len(METHOD_ORDER)):
|
||||
ax.text(j, i, f"{ari_matrix[i, j]:.2f}", ha="center", va="center",
|
||||
color="white" if ari_matrix[i, j] < 0.6 else "black", fontsize=11)
|
||||
ax.set_title("Patient-grouping agreement (ARI)\nIQ-OTH/NCCD, 1,097 images")
|
||||
fig.colorbar(im, ax=ax, label="Adjusted Rand Index")
|
||||
plt.tight_layout()
|
||||
os.makedirs(PLOTS_DIR, exist_ok=True)
|
||||
plot_path = os.path.join(PLOTS_DIR, "grouping_agreement_heatmap.png")
|
||||
plt.savefig(plot_path, dpi=150)
|
||||
plt.close()
|
||||
print(f"Saved heatmap → {plot_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
verify_feature_dims.py
|
||||
|
||||
Quick check: what does TF/Keras actually output for each model
|
||||
with include_top=False? Compare against manuscript claims.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
from tensorflow.keras.applications import (
|
||||
VGG16, DenseNet121, EfficientNetB1, MobileNetV2, ResNet50
|
||||
)
|
||||
|
||||
# Match the manuscript's input sizes
|
||||
models = {
|
||||
"VGG16": (VGG16, 224),
|
||||
"DenseNet121": (DenseNet121, 224),
|
||||
"EfficientNetB1":(EfficientNetB1, 240),
|
||||
"MobileNetV2": (MobileNetV2, 224),
|
||||
"ResNet50": (ResNet50, 224),
|
||||
}
|
||||
|
||||
manuscript_claims = {
|
||||
"VGG16": 25088,
|
||||
"DenseNet121": 50176,
|
||||
"EfficientNetB1":62720,
|
||||
"MobileNetV2": 62720,
|
||||
"ResNet50": 100352,
|
||||
}
|
||||
|
||||
print(f"{'Model':<18s} {'Input':>5s} {'Spatial':>10s} {'Flattened':>10s} {'Manuscript':>12s} {'Match?':>7s}")
|
||||
print("-" * 70)
|
||||
|
||||
for name, (model_fn, input_size) in models.items():
|
||||
model = model_fn(weights="imagenet", include_top=False,
|
||||
input_shape=(input_size, input_size, 3))
|
||||
|
||||
# Pass a dummy batch through
|
||||
dummy = np.random.randn(1, input_size, input_size, 3)
|
||||
# Need to preprocess correctly — but shape doesn't depend on values
|
||||
output = model.predict(dummy, verbose=0)
|
||||
|
||||
spatial = output.shape[1:4] # (H, W, C) for channels_last
|
||||
flattened = int(np.prod(spatial))
|
||||
claimed = manuscript_claims[name]
|
||||
match = "✅" if flattened == claimed else "❌"
|
||||
|
||||
print(f"{name:<18s} {input_size:>4}d {str(spatial):>10s} {flattened:>10d} {claimed:>12d} {match:>7s}")
|
||||
|
||||
print("\nNote: TF uses channels_last (NHWC) format.")
|
||||
Reference in New Issue
Block a user