rpotter6298
85 lines
3.0 KiB
Python
85 lines
3.0 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
classification.py — single-seed image-level vs patient-level comparison.
|
|
|
|
This script OWNS the shared classification-runs cache
|
|
(scripts/cache/classification_runs.json): a flat list of per-(model, seed,
|
|
split) run dicts. It reads any runs it needs from the cache and writes back any
|
|
it has to compute, so the cache is maintained here. figure4_6.py references the
|
|
same file to draw the 20-seed boxplots.
|
|
|
|
Usage:
|
|
conda activate fundus_imaging
|
|
python scripts/classification.py
|
|
"""
|
|
|
|
import os, sys, json
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(
|
|
os.path.abspath(__file__)))))
|
|
|
|
from classes import PatientLeakageClassifier
|
|
|
|
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
ROOT = os.path.dirname(SCRIPT_DIR)
|
|
MODELS = ["VGG16", "DenseNet121", "EfficientNetB1", "MobileNetV2", "ResNet50"]
|
|
SEED = 20
|
|
CACHE_PATH = os.path.join(SCRIPT_DIR, "cache", "classification_runs.json")
|
|
|
|
# Load the run cache: keep the raw list (to append to) and an index (to look up).
|
|
runs = []
|
|
if os.path.exists(CACHE_PATH):
|
|
with open(CACHE_PATH) as f:
|
|
runs = json.load(f)
|
|
index = {(r["model"], r["seed"], r["split_type"]): r for r in runs}
|
|
|
|
clf = None # lazily created only if the cache is missing something
|
|
|
|
|
|
def get_run(name, split):
|
|
"""Return the cached run, else compute it and write it back to the cache."""
|
|
global clf
|
|
hit = index.get((name, SEED, split))
|
|
if hit is not None:
|
|
return hit
|
|
if clf is None:
|
|
clf = PatientLeakageClassifier(
|
|
os.path.join(ROOT, "results", "simple_patient_manifest.csv"),
|
|
os.path.join(ROOT, "features"), n_jobs=8)
|
|
print(f" (cache miss for {name}/{split} — computing)")
|
|
r = clf.run(name, SEED, split)
|
|
runs.append(r)
|
|
index[(name, SEED, split)] = r
|
|
os.makedirs(os.path.dirname(CACHE_PATH), exist_ok=True)
|
|
with open(CACHE_PATH, "w") as f:
|
|
json.dump(runs, f, indent=2)
|
|
return r
|
|
|
|
|
|
print(f"{'='*60}")
|
|
print(f"IMAGE-LEVEL vs PATIENT-LEVEL (seed={SEED})")
|
|
print(f"{'='*60}")
|
|
|
|
results = []
|
|
for name in MODELS:
|
|
print(f"\n {name} ...")
|
|
img = get_run(name, "image")
|
|
pat = get_run(name, "patient")
|
|
results.append({"model": name,
|
|
"image_cv": img["cv"], "image_test": img["test"],
|
|
"patient_cv": pat["cv"], "patient_test": pat["test"],
|
|
"drop": img["test"] - pat["test"]})
|
|
print(f" Image: CV={img['cv']:.4f} Test={img['test']:.4f}")
|
|
print(f" Patient: CV={pat['cv']:.4f} Test={pat['test']:.4f}")
|
|
print(f" Drop: {img['test'] - pat['test']:.4f}")
|
|
|
|
print(f"\n {'Model':<18s} {'Img-CV':>8s} {'Img-Test':>9s} "
|
|
f"{'Pat-CV':>8s} {'Pat-Test':>9s} {'Drop':>7s}")
|
|
print(f" {'-'*54}")
|
|
for r in results:
|
|
print(f" {r['model']:<18s} {r['image_cv']:>8.4f} {r['image_test']:>9.4f} "
|
|
f"{r['patient_cv']:>8.4f} {r['patient_test']:>9.4f} {r['drop']:>7.4f}")
|
|
|
|
with open(os.path.join(ROOT, "results", "classification_pca50.json"), "w") as f:
|
|
json.dump(results, f, indent=2)
|
|
print(f"\nDONE")
|