rpotter6298
250 lines
8.4 KiB
Python
Executable File
250 lines
8.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Compare suspect AUC from image tower vs crop-derived geometry (CDR)."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Tuple
|
|
|
|
import sys
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
from PIL import Image
|
|
from sklearn.metrics import roc_auc_score
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
if str(REPO_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(REPO_ROOT))
|
|
|
|
from classes import build_papila_clinical
|
|
from classes.hypertower import UNetImageCropper, ManifestImageCropper
|
|
|
|
|
|
# ---------------------------
|
|
# Config (edit in IDE)
|
|
# ---------------------------
|
|
RUN_DIRS = [
|
|
Path("analysis_data/1030_Balanced_Unet_Perimg_Resnet_SE16NormB_SE16NormT_multi_fused/1030_Balanced_Unet_Perimg_Resnet_SE16NormB_SE16NormT_multi_fused_20251030_091842"),
|
|
Path("analysis_data/1030_Balanced_GT_Perimg_Resnet_SE16NormB_SE16NormT_multi_fused/1030_Balanced_GT_Perimg_Resnet_SE16NormB_SE16NormT_multi_fused_20251030_113730"),
|
|
]
|
|
GEOM_CACHE_ROOT = Path("analysis_data/geometry_cache")
|
|
SUSPECT_LABEL = 2
|
|
|
|
|
|
def _load_json(path: Path) -> Dict:
|
|
if not path.exists():
|
|
return {}
|
|
try:
|
|
return json.loads(path.read_text())
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def _drop_holdout_rows(clinical, holdout_path: Path) -> None:
|
|
if not holdout_path.exists():
|
|
return
|
|
holdout = pd.read_csv(holdout_path)
|
|
if holdout.empty:
|
|
return
|
|
if "Patient ID" not in holdout.columns or "eyeID" not in holdout.columns:
|
|
return
|
|
holdout_keys = set(zip(holdout["Patient ID"].astype(int), holdout["eyeID"].astype(str)))
|
|
df = clinical.df.copy()
|
|
df["_key"] = list(zip(df["Patient ID"].astype(int), df["eyeID"].astype(str)))
|
|
df = df[~df["_key"].isin(holdout_keys)].drop(columns=["_key"]).reset_index(drop=True)
|
|
|
|
clinical.frames = [df.copy()]
|
|
clinical.df = df.copy()
|
|
clinical._infer_or_validate_feature_types()
|
|
clinical._compute_numeric_stats()
|
|
clinical._build_cat_maps()
|
|
clinical._compute_feature_dim()
|
|
clinical._build_kfold_indices()
|
|
|
|
|
|
def _make_cropper(args: Dict, cache_dir: Path):
|
|
manifest = args.get("img_crop_manifest")
|
|
if not manifest:
|
|
raise RuntimeError("img_crop_manifest missing; cannot compute geometry features.")
|
|
|
|
scale = float(args.get("img_crop_scale", 2.5))
|
|
target_size = int(args.get("img_crop_size", 224))
|
|
use_gt = bool(args.get("img_crop_gt", False))
|
|
|
|
if use_gt:
|
|
return ManifestImageCropper(
|
|
manifest_path=Path(manifest),
|
|
scale=scale,
|
|
target_size=target_size,
|
|
cache_dir=cache_dir,
|
|
)
|
|
|
|
weights = args.get("img_crop_weights")
|
|
if not weights:
|
|
raise RuntimeError("img_crop_weights missing for UNet cropper.")
|
|
|
|
normalize = args.get("img_crop_normalize", "per_image")
|
|
threshold = float(args.get("img_crop_threshold", 0.5))
|
|
tta = bool(args.get("img_crop_tta", False))
|
|
return UNetImageCropper(
|
|
manifest_path=Path(manifest),
|
|
weights_path=Path(weights),
|
|
normalize=normalize,
|
|
threshold=threshold,
|
|
tta=tta,
|
|
scale=scale,
|
|
target_size=target_size,
|
|
cache_dir=cache_dir,
|
|
)
|
|
|
|
|
|
def _geometry_scores(
|
|
clinical,
|
|
cropper,
|
|
test_df: pd.DataFrame,
|
|
) -> Tuple[np.ndarray, np.ndarray]:
|
|
scores: List[float] = []
|
|
keep_mask: List[bool] = []
|
|
for _, row in test_df.iterrows():
|
|
img_path = clinical.get_image_path(row)
|
|
try:
|
|
image = Image.open(img_path).convert("RGB")
|
|
except Exception:
|
|
scores.append(float("nan"))
|
|
keep_mask.append(False)
|
|
continue
|
|
feats = cropper.geometry_features(image, img_path)
|
|
if feats is None or len(feats) == 0:
|
|
scores.append(float("nan"))
|
|
keep_mask.append(False)
|
|
else:
|
|
scores.append(float(feats[0])) # area_ratio (CDR)
|
|
keep_mask.append(True)
|
|
return np.asarray(scores, dtype=float), np.asarray(keep_mask, dtype=bool)
|
|
|
|
|
|
def _suspect_auc(y_true: np.ndarray, scores: np.ndarray) -> float:
|
|
y = (y_true == SUSPECT_LABEL).astype(int)
|
|
if y.sum() == 0 or y.sum() == len(y):
|
|
return float("nan")
|
|
return float(roc_auc_score(y, scores))
|
|
|
|
|
|
def main() -> None:
|
|
rows: List[Dict[str, object]] = []
|
|
|
|
for run_dir in RUN_DIRS:
|
|
cli_path = run_dir / "cli_args.json"
|
|
cli_args = _load_json(cli_path)
|
|
if not cli_args:
|
|
print(f"[warn] Missing cli_args.json in {run_dir}")
|
|
continue
|
|
|
|
label_col = cli_args.get("label_col", "Diagnosis")
|
|
cat_cols = cli_args.get("cat_cols", ["Gender", "Phakic/Pseudophakic"])
|
|
n_splits = int(cli_args.get("n_splits", 5))
|
|
fold_seed = int(cli_args.get("fold_seed", 42))
|
|
eval_mode = str(cli_args.get("eval_mode", "multiclass")).lower()
|
|
|
|
clinical = build_papila_clinical(
|
|
image_dir=cli_args.get("image_dir", "Papila/FundusImages"),
|
|
clinical_dir=cli_args.get("clinical_dir", "Papila/ClinicalData"),
|
|
label_col=label_col,
|
|
cat_cols=cat_cols,
|
|
n_splits=n_splits,
|
|
random_seed=fold_seed,
|
|
)
|
|
|
|
if eval_mode == "binary":
|
|
clinical.df = clinical.df[clinical.df[label_col].isin([0, 1])].reset_index(drop=True)
|
|
clinical.frames = [clinical.df.copy()]
|
|
clinical._infer_or_validate_feature_types()
|
|
clinical._compute_numeric_stats()
|
|
clinical._build_cat_maps()
|
|
clinical._compute_feature_dim()
|
|
clinical._build_kfold_indices()
|
|
|
|
_drop_holdout_rows(clinical, run_dir / "holdout.csv")
|
|
|
|
cache_dir = GEOM_CACHE_ROOT / run_dir.name
|
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
cropper = _make_cropper(cli_args, cache_dir=cache_dir)
|
|
|
|
all_geom_scores: List[float] = []
|
|
all_img_scores: List[float] = []
|
|
all_y: List[int] = []
|
|
|
|
for fold in range(n_splits):
|
|
y_path = run_dir / f"fold{fold}_y_true.npy"
|
|
p_img_path = run_dir / f"fold{fold}_probs_img.npy"
|
|
if not y_path.exists() or not p_img_path.exists():
|
|
continue
|
|
|
|
y_true = np.load(y_path)
|
|
probs_img = np.load(p_img_path)
|
|
if probs_img.ndim != 2 or probs_img.shape[1] <= SUSPECT_LABEL:
|
|
continue
|
|
|
|
_, test_df = clinical.get_split_dfs(fold)
|
|
if len(test_df) != len(y_true):
|
|
print(
|
|
f"[warn] {run_dir.name} fold{fold}: test_df len {len(test_df)} != y_true len {len(y_true)}"
|
|
)
|
|
|
|
geom_scores, keep_mask = _geometry_scores(clinical, cropper, test_df)
|
|
if keep_mask.sum() == 0:
|
|
print(f"[warn] {run_dir.name} fold{fold}: no valid geometry features")
|
|
continue
|
|
|
|
y_fold = y_true[: len(geom_scores)][keep_mask]
|
|
geom_fold = geom_scores[keep_mask]
|
|
img_fold = probs_img[: len(geom_scores), SUSPECT_LABEL][keep_mask]
|
|
|
|
geom_auc = _suspect_auc(y_fold, geom_fold)
|
|
img_auc = _suspect_auc(y_fold, img_fold)
|
|
|
|
rows.append(
|
|
{
|
|
"run": run_dir.name,
|
|
"fold": fold,
|
|
"metric": "suspect_auc",
|
|
"image_auc": img_auc,
|
|
"geometry_auc": geom_auc,
|
|
"n": int(len(y_fold)),
|
|
}
|
|
)
|
|
|
|
all_geom_scores.append(geom_fold)
|
|
all_img_scores.append(img_fold)
|
|
all_y.append(y_fold)
|
|
|
|
if all_y:
|
|
y_all = np.concatenate(all_y)
|
|
geom_all = np.concatenate(all_geom_scores)
|
|
img_all = np.concatenate(all_img_scores)
|
|
rows.append(
|
|
{
|
|
"run": run_dir.name,
|
|
"fold": "all",
|
|
"metric": "suspect_auc",
|
|
"image_auc": _suspect_auc(y_all, img_all),
|
|
"geometry_auc": _suspect_auc(y_all, geom_all),
|
|
"n": int(len(y_all)),
|
|
}
|
|
)
|
|
|
|
if not rows:
|
|
raise SystemExit("No results produced; check run paths and files.")
|
|
|
|
df = pd.DataFrame(rows)
|
|
out_path = GEOM_CACHE_ROOT / "suspect_auc_geometry_vs_image.csv"
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
df.to_csv(out_path, index=False)
|
|
print(df.to_string(index=False, float_format=lambda x: f"{x:.4f}"))
|
|
print(f"\nSaved: {out_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|