Add scripts for patient classification and visualization
- Created `classification.py` for comparing image-level and patient-level classification results using various CNN models. - Implemented `create_patient_groups.py` to extract features, generate PCA/t-SNE plots, and identify patient groups via K-means clustering. - Added `figure6.py` to generate boxplots for test accuracy across multiple seeds. - Developed `simple_patient_tsne.py` to perform t-SNE visualization of patient groups and save results in a manifest file. - Introduced `simple_patient_manifest.csv` to store patient IDs, classes, image counts, and associated images.
This commit is contained in:
rpotter6298
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
classification.py — single-seed image-level vs patient-level comparison.
|
||||
|
||||
Usage:
|
||||
conda activate fundus_imaging
|
||||
python scripts/classification.py
|
||||
"""
|
||||
|
||||
import os, sys, json
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from classes import PatientLeakageClassifier
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
MODELS = ["VGG16", "DenseNet121", "EfficientNetB1", "MobileNetV2", "ResNet50"]
|
||||
SEED = 20
|
||||
|
||||
clf = PatientLeakageClassifier(
|
||||
os.path.join(ROOT, "results", "simple_patient_manifest.csv"),
|
||||
os.path.join(ROOT, "features"),
|
||||
n_jobs=8)
|
||||
|
||||
print(f"{'='*60}")
|
||||
print(f"IMAGE-LEVEL vs PATIENT-LEVEL (seed={SEED})")
|
||||
print(f"{'='*60}")
|
||||
|
||||
results = []
|
||||
for name in MODELS:
|
||||
print(f"\n {name} ...")
|
||||
img = clf.run(name, SEED, "image")
|
||||
pat = clf.run(name, SEED, "patient")
|
||||
results.append({"model": name,
|
||||
"image_cv": img["cv"], "image_test": img["test"],
|
||||
"patient_cv": pat["cv"], "patient_test": pat["test"],
|
||||
"drop": img["test"] - pat["test"]})
|
||||
print(f" Image: CV={img['cv']:.4f} Test={img['test']:.4f}")
|
||||
print(f" Patient: CV={pat['cv']:.4f} Test={pat['test']:.4f}")
|
||||
print(f" Drop: {img['test'] - pat['test']:.4f}")
|
||||
|
||||
print(f"\n {'Model':<18s} {'Img-CV':>8s} {'Img-Test':>9s} "
|
||||
f"{'Pat-CV':>8s} {'Pat-Test':>9s} {'Drop':>7s}")
|
||||
print(f" {'-'*54}")
|
||||
for r in results:
|
||||
print(f" {r['model']:<18s} {r['image_cv']:>8.4f} {r['image_test']:>9.4f} "
|
||||
f"{r['patient_cv']:>8.4f} {r['patient_test']:>9.4f} {r['drop']:>7.4f}")
|
||||
|
||||
with open(os.path.join(ROOT, "results", "classification_results.json"), "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
print(f"\nDONE")
|
||||
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
create_patient_groups.py
|
||||
|
||||
End-to-end pipeline for all five CNNs from the manuscript:
|
||||
VGG16, DenseNet121, EfficientNetB1, MobileNetV2, ResNet50
|
||||
|
||||
For each model:
|
||||
1. Extract deep features and save to features/{Model}_features.npz
|
||||
2. Generate PCA / t-SNE plot → plots/{Model}_pca_tsne.png
|
||||
3. Estimate patient groups via K-means → features/{Model}_patient_groups.npy
|
||||
|
||||
Usage:
|
||||
conda activate fundus_imaging
|
||||
python scripts/create_patient_groups.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from classes import FeatureExtractor, PatientIdentifier
|
||||
from classes.visualizations import plot_pca_tsne
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
BASE_PATH = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
|
||||
"The IQ-OTHNCCD lung cancer dataset"
|
||||
)
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
FEATURES_DIR = os.path.join(PROJECT_ROOT, "features")
|
||||
PLOTS_DIR = os.path.join(PROJECT_ROOT, "plots")
|
||||
|
||||
os.makedirs(FEATURES_DIR, exist_ok=True)
|
||||
os.makedirs(PLOTS_DIR, exist_ok=True)
|
||||
|
||||
MODELS = ["VGG16", "DenseNet121", "EfficientNetB1", "MobileNetV2", "ResNet50"]
|
||||
|
||||
PATIENT_ESTIMATES = {
|
||||
"Bengin cases": 15,
|
||||
"Malignant cases": 40,
|
||||
"Normal cases": 55,
|
||||
}
|
||||
|
||||
LEGEND_NAMES = {
|
||||
"Bengin cases": "Benign",
|
||||
"Malignant cases": "Malignant",
|
||||
"Normal cases": "Normal",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Run pipeline for each model
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
for model_name in MODELS:
|
||||
print("\n" + "=" * 60)
|
||||
print(f"MODEL: {model_name}")
|
||||
print("=" * 60)
|
||||
|
||||
# --- Step 1: Extract features ---
|
||||
print("\n [1/3] Feature extraction ...")
|
||||
extractor = FeatureExtractor(model_name=model_name)
|
||||
features_path = os.path.join(FEATURES_DIR, f"{model_name}_features.npz")
|
||||
|
||||
if os.path.exists(features_path):
|
||||
print(f" Loading cached features from {features_path}")
|
||||
X, Y, filenames = FeatureExtractor.load_features(features_path)
|
||||
else:
|
||||
X, Y, filenames = extractor.extract(BASE_PATH)
|
||||
extractor.save_features(X, Y, filenames, FEATURES_DIR)
|
||||
|
||||
print(f" {model_name}: X shape = {X.shape}")
|
||||
|
||||
# --- Step 2: PCA / t-SNE ---
|
||||
print(f"\n [2/3] PCA / t-SNE visualization ...")
|
||||
plot_path = os.path.join(PLOTS_DIR, f"{model_name}_pca_tsne.png")
|
||||
plot_pca_tsne(X, Y, legend_names=LEGEND_NAMES, output_path=plot_path)
|
||||
|
||||
# --- Step 3: Patient identification ---
|
||||
print(f"\n [3/3] Patient identification (K-means) ...")
|
||||
identifier = PatientIdentifier(patient_estimates=PATIENT_ESTIMATES)
|
||||
groups = identifier.identify(BASE_PATH, filenames, Y)
|
||||
|
||||
groups_path = os.path.join(FEATURES_DIR, f"{model_name}_patient_groups.npy")
|
||||
np.save(groups_path, groups)
|
||||
print(f" Saved patient groups → {groups_path}")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("ALL MODELS COMPLETE")
|
||||
print("=" * 60)
|
||||
for model_name in MODELS:
|
||||
fp = os.path.join(FEATURES_DIR, f"{model_name}_features.npz")
|
||||
gp = os.path.join(FEATURES_DIR, f"{model_name}_patient_groups.npy")
|
||||
pp = os.path.join(PLOTS_DIR, f"{model_name}_pca_tsne.png")
|
||||
print(f" {model_name:18s} features: {os.path.basename(fp):30s} groups: {os.path.basename(gp)}")
|
||||
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
figure6.py — test accuracy boxplots across 20 seeds (Figure 6).
|
||||
|
||||
Usage:
|
||||
conda activate fundus_imaging
|
||||
python scripts/figure6.py
|
||||
"""
|
||||
|
||||
import os, sys, json
|
||||
import numpy as np
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.patches import Patch
|
||||
from tqdm import tqdm
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from classes import PatientLeakageClassifier
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
MODELS = ["VGG16", "DenseNet121", "EfficientNetB1", "MobileNetV2", "ResNet50"]
|
||||
SEEDS = list(range(1, 21))
|
||||
|
||||
clf = PatientLeakageClassifier(
|
||||
os.path.join(ROOT, "results", "simple_patient_manifest.csv"),
|
||||
os.path.join(ROOT, "features"),
|
||||
n_jobs=8)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Run all seeds
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
all_data = [] # flat list of per-run dicts for raw data file
|
||||
|
||||
for name in tqdm(MODELS, desc="Models"):
|
||||
for seed in tqdm(SEEDS, desc=f" {name} seeds", leave=False):
|
||||
for stype in ["image", "patient"]:
|
||||
r = clf.run(name, seed, stype)
|
||||
all_data.append(r)
|
||||
|
||||
# Save raw data
|
||||
with open(os.path.join(ROOT, "results", "figure6_data.json"), "w") as f:
|
||||
json.dump(all_data, f, indent=2)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Build per-model accuracy lists
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
accs = {} # model -> {image: [accs], patient: [accs]}
|
||||
for name in MODELS:
|
||||
accs[name] = {"image": [], "patient": []}
|
||||
for r in all_data:
|
||||
accs[r["model"]][r["split_type"]].append(r["test"])
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plot
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
fig, ax = plt.subplots(1, 1, figsize=(10, 6))
|
||||
|
||||
for i, name in enumerate(MODELS):
|
||||
pos_img = i * 2 + 0.7
|
||||
pos_pat = i * 2 + 1.3
|
||||
|
||||
for pos, stype, color in [(pos_img, "image", '#4C9BD4'),
|
||||
(pos_pat, "patient", '#6DBF6D')]:
|
||||
data = accs[name][stype]
|
||||
bp = ax.boxplot(data, positions=[pos], widths=0.5,
|
||||
patch_artist=True, showfliers=True,
|
||||
flierprops=dict(marker='o', markersize=3))
|
||||
bp['boxes'][0].set_facecolor(color)
|
||||
med = np.median(data)
|
||||
ax.annotate(f"{med:.3f}", (pos, med), fontsize=6,
|
||||
ha='center', va='bottom')
|
||||
|
||||
ax.legend(handles=[
|
||||
Patch(facecolor='#4C9BD4', label='Image-level split'),
|
||||
Patch(facecolor='#6DBF6D', label='Patient-level split'),
|
||||
], loc='lower right')
|
||||
|
||||
ax.set_xticks([p + 1 for p in range(0, len(MODELS) * 2, 2)])
|
||||
ax.set_xticklabels(MODELS)
|
||||
ax.set_ylabel("Test accuracy")
|
||||
ax.set_title("Figure 6 — Image-level vs Patient-level test accuracy (20 seeds)")
|
||||
ax.set_ylim(0.7, 1.02)
|
||||
ax.grid(axis='y', alpha=0.3)
|
||||
|
||||
plt.tight_layout()
|
||||
out = os.path.join(ROOT, "plots", "figure6.png")
|
||||
plt.savefig(out, dpi=150)
|
||||
plt.close()
|
||||
|
||||
print(f"\nSaved → {out}")
|
||||
print(f"Saved → {ROOT}/results/figure6_data.json")
|
||||
print("DONE")
|
||||
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
simple_patient_tsne.py
|
||||
|
||||
The short path:
|
||||
1. Load VGG16 features
|
||||
2. K-means per class (15/40/55 patients) in PCA-50d space
|
||||
3. Optional: iterative centroid refinement
|
||||
4. Plot t-SNE colored by cluster, with centroid labels
|
||||
|
||||
Usage:
|
||||
conda activate fundus_imaging
|
||||
python scripts/simple_patient_tsne.py
|
||||
"""
|
||||
|
||||
import os, sys, re, csv, json
|
||||
from collections import defaultdict
|
||||
import numpy as np
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.decomposition import PCA
|
||||
from sklearn.manifold import TSNE
|
||||
from sklearn.cluster import KMeans
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
FEATURES_DIR = os.path.join(PROJECT_ROOT, "features")
|
||||
PLOTS_DIR = os.path.join(PROJECT_ROOT, "plots")
|
||||
RESULTS_DIR = os.path.join(PROJECT_ROOT, "results")
|
||||
os.makedirs(PLOTS_DIR, exist_ok=True)
|
||||
os.makedirs(RESULTS_DIR, exist_ok=True)
|
||||
|
||||
PATIENT_COUNTS = {"Bengin cases": 15, "Malignant cases": 40, "Normal cases": 55}
|
||||
CLASS_NAMES = {"Bengin cases": "Benign", "Malignant cases": "Malignant", "Normal cases": "Normal"}
|
||||
RANDOM_STATE = 42
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Load VGG16 features
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
print("Loading VGG16 features ...")
|
||||
data = np.load(os.path.join(FEATURES_DIR, "VGG16_features.npz"), allow_pickle=True)
|
||||
X, Y, filenames = data["X"], data["Y"], data["filenames"]
|
||||
|
||||
def f2n(fname):
|
||||
m = re.search(r'\((\d+)\)', fname)
|
||||
num = int(m.group(1)) if m else None
|
||||
for cls_key, prefix in [("Bengin cases","B"),("Malignant cases","M"),("Normal cases","N")]:
|
||||
if fname.startswith(cls_key.rstrip("s")):
|
||||
return f"{prefix}_{num:03d}" if num else fname
|
||||
return fname
|
||||
|
||||
img_nums = np.array([f2n(f) for f in filenames])
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. K-means per class in PCA-50d space
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
print("Clustering in PCA-50d space ...")
|
||||
n_pca = min(50, X.shape[0] - 1, X.shape[1])
|
||||
X_pca = PCA(n_components=n_pca, random_state=RANDOM_STATE).fit_transform(X)
|
||||
|
||||
image_to_patient = {}
|
||||
patient_to_images = defaultdict(list)
|
||||
|
||||
for class_name, k in PATIENT_COUNTS.items():
|
||||
mask = Y == class_name
|
||||
X_class = X_pca[mask]
|
||||
idx_class = np.where(mask)[0]
|
||||
|
||||
kmeans = KMeans(n_clusters=k, random_state=RANDOM_STATE, n_init=20)
|
||||
labels = kmeans.fit_predict(X_class)
|
||||
|
||||
prefix = {"Bengin cases": "Benign", "Malignant cases": "Malignant", "Normal cases": "Normal"}[class_name]
|
||||
for i, cluster_id in enumerate(labels):
|
||||
pid = f"{prefix}_{cluster_id:02d}"
|
||||
img = img_nums[idx_class[i]]
|
||||
image_to_patient[img] = pid
|
||||
patient_to_images[pid].append(img)
|
||||
|
||||
print(f" {len(patient_to_images)} patients, {len(image_to_patient)} images")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. Iterative centroid refinement (optional, 3 passes)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
print("Refining assignments (nearest-centroid, 5 passes) ...")
|
||||
for iteration in range(5):
|
||||
# Compute centroids
|
||||
centroids = {}
|
||||
for pid, imgs in patient_to_images.items():
|
||||
idxs = [np.where(img_nums == img)[0][0] for img in imgs]
|
||||
centroids[pid] = X_pca[idxs].mean(axis=0)
|
||||
|
||||
# Reassign
|
||||
moves = 0
|
||||
for class_name in PATIENT_COUNTS:
|
||||
mask = Y == class_name
|
||||
for i in np.where(mask)[0]:
|
||||
img = img_nums[i]
|
||||
old_pid = image_to_patient[img]
|
||||
# Find nearest centroid in same class
|
||||
best_pid = old_pid
|
||||
best_dist = float('inf')
|
||||
for pid, c in centroids.items():
|
||||
if pid.startswith(CLASS_NAMES[class_name]):
|
||||
d = float(np.linalg.norm(X_pca[i] - c))
|
||||
if d < best_dist:
|
||||
best_dist = d
|
||||
best_pid = pid
|
||||
if best_pid != old_pid:
|
||||
patient_to_images[old_pid].remove(img)
|
||||
patient_to_images[best_pid].append(img)
|
||||
image_to_patient[img] = best_pid
|
||||
moves += 1
|
||||
print(f" Pass {iteration+1}: {moves} moves")
|
||||
if moves == 0:
|
||||
break
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. t-SNE
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
print("Computing t-SNE ...")
|
||||
X_tsne = TSNE(n_components=2, perplexity=35, learning_rate="auto",
|
||||
init="pca", random_state=RANDOM_STATE).fit_transform(X_pca)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. Plot — one figure per class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
for class_name, display_name in CLASS_NAMES.items():
|
||||
fig, ax = plt.subplots(1, 1, figsize=(14, 10))
|
||||
ax.scatter(X_tsne[:, 0], X_tsne[:, 1], c="lightgray", s=3, alpha=0.15)
|
||||
|
||||
mask = Y == class_name
|
||||
class_pids = sorted([p for p in patient_to_images if p.startswith(display_name)])
|
||||
n_patients = len(class_pids)
|
||||
cmap = plt.cm.tab20 if n_patients <= 20 else plt.cm.gist_ncar
|
||||
|
||||
for pi, pid in enumerate(class_pids):
|
||||
color = cmap(pi % 20) if n_patients <= 20 else cmap(pi / max(n_patients-1, 1))
|
||||
pts_x, pts_y = [], []
|
||||
for img in patient_to_images[pid]:
|
||||
i = np.where(img_nums == img)[0][0]
|
||||
pts_x.append(X_tsne[i, 0])
|
||||
pts_y.append(X_tsne[i, 1])
|
||||
ax.scatter(pts_x, pts_y, c=[color], s=18, alpha=0.8, label='_nolegend_')
|
||||
# Centroid diamond (no label)
|
||||
cx, cy = np.mean(pts_x), np.mean(pts_y)
|
||||
ax.scatter(cx, cy, c=[color], s=60, marker='D', edgecolors='black',
|
||||
linewidths=0.6, zorder=5, label='_nolegend_')
|
||||
|
||||
# Legend elements
|
||||
from matplotlib.lines import Line2D
|
||||
legend_elements = [
|
||||
Line2D([0], [0], marker='o', color='w', markerfacecolor='gray',
|
||||
markersize=8, label='Patient images (dots)'),
|
||||
Line2D([0], [0], marker='D', color='w', markerfacecolor='gray',
|
||||
markersize=8, label='Patient centroids (diamonds)'),
|
||||
]
|
||||
ax.legend(handles=legend_elements, loc='lower right')
|
||||
|
||||
ax.set_title(f"{display_name} — VGG16 t-SNE ({n_patients} patients)")
|
||||
ax.set_xlabel("t-SNE dim 1")
|
||||
ax.set_ylabel("t-SNE dim 2")
|
||||
plt.tight_layout()
|
||||
out = os.path.join(PLOTS_DIR, f"tsne_{display_name}.png")
|
||||
plt.savefig(out, dpi=150)
|
||||
plt.close()
|
||||
print(f" Saved {out}")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 6. Save assignments
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
manifest = []
|
||||
for pid in sorted(patient_to_images.keys()):
|
||||
imgs = sorted(patient_to_images[pid])
|
||||
manifest.append({"patient_id": pid, "class": pid.split("_")[0],
|
||||
"n_images": len(imgs), "images": ";".join(imgs)})
|
||||
|
||||
with open(os.path.join(RESULTS_DIR, "simple_patient_manifest.csv"), "w", newline="") as f:
|
||||
w = csv.DictWriter(f, fieldnames=["patient_id", "class", "n_images", "images"])
|
||||
w.writeheader()
|
||||
w.writerows(manifest)
|
||||
|
||||
print(f"\nSaved simple_patient_manifest.csv ({len(manifest)} patients, "
|
||||
f"{sum(m['n_images'] for m in manifest)} images)")
|
||||
print("DONE")
|
||||
Reference in New Issue
Block a user