moved_repo_first_update
This commit is contained in:
rpotter6298
Executable
+757
@@ -0,0 +1,757 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Basic analytics helpers for PAPILA clinical data."""
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Iterable, List, Tuple
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.base import clone
|
||||
from sklearn.metrics import roc_curve, auc, roc_auc_score, accuracy_score
|
||||
from sklearn.ensemble import RandomForestClassifier
|
||||
from sklearn.model_selection import StratifiedKFold
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.neighbors import KNeighborsClassifier
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
from sklearn.svm import SVC
|
||||
|
||||
from classes import build_papila_clinical
|
||||
|
||||
|
||||
class basic_analytics:
|
||||
def __init__(
|
||||
self,
|
||||
image_dir: str = "Papila/FundusImages",
|
||||
clinical_dir: str = "Papila/ClinicalData",
|
||||
label_col: str = "Diagnosis",
|
||||
cat_cols: Iterable[str] | None = None,
|
||||
exclude_cols: Iterable[str] | None = None,
|
||||
positive_label: int = 1,
|
||||
negative_label: int = 0,
|
||||
drop_labels: Iterable[int] = (2,),
|
||||
output_dir: Path | str = Path("analysis_data/basic_analysis"),
|
||||
debug: bool = False,
|
||||
) -> None:
|
||||
self.image_dir = image_dir
|
||||
self.clinical_dir = clinical_dir
|
||||
self.label_col = label_col
|
||||
self.cat_cols = (
|
||||
list(cat_cols)
|
||||
if cat_cols is not None
|
||||
else ["Gender", "Phakic/Pseudophakic"]
|
||||
)
|
||||
base_excludes = {"Pneumatic", "Perkins"}
|
||||
self.exclude_cols = base_excludes | set(exclude_cols or [])
|
||||
self.positive_label = positive_label
|
||||
self.negative_label = negative_label
|
||||
self.drop_labels = list(drop_labels or [])
|
||||
self.output_dir = Path(output_dir)
|
||||
self.debug = debug
|
||||
|
||||
@staticmethod
|
||||
def _sanitize(name: str) -> str:
|
||||
safe = re.sub(r"[^A-Za-z0-9._-]+", "_", str(name)).strip("_")
|
||||
return safe or "var"
|
||||
|
||||
def _build_clinical(self):
|
||||
return build_papila_clinical(
|
||||
image_dir=self.image_dir,
|
||||
clinical_dir=self.clinical_dir,
|
||||
label_col=self.label_col,
|
||||
cat_cols=self.cat_cols,
|
||||
)
|
||||
|
||||
def _select_binary_labels(
|
||||
self,
|
||||
labels: pd.Series,
|
||||
) -> Tuple[np.ndarray, np.ndarray]:
|
||||
labels_num = pd.to_numeric(labels, errors="coerce")
|
||||
use_num = labels_num.notna().any()
|
||||
lab = labels_num if use_num else labels.astype(str)
|
||||
|
||||
drop_set = set(self.drop_labels or [])
|
||||
keep = lab.isin([self.positive_label, self.negative_label])
|
||||
if drop_set:
|
||||
keep &= ~lab.isin(drop_set)
|
||||
|
||||
y = (lab == self.positive_label).astype(int)
|
||||
return y.values, keep.values
|
||||
|
||||
def _base_exclude(self) -> set:
|
||||
base_exclude = (
|
||||
{self.label_col, "Patient ID"} | self.exclude_cols | set(self.cat_cols)
|
||||
)
|
||||
if "eyeID" not in self.cat_cols:
|
||||
base_exclude.add("eyeID")
|
||||
return base_exclude
|
||||
|
||||
def _numeric_columns(self, df: pd.DataFrame) -> List[str]:
|
||||
base_exclude = self._base_exclude()
|
||||
candidate_cols = [c for c in df.columns if c not in base_exclude]
|
||||
numeric_cols: List[str] = []
|
||||
for col in candidate_cols:
|
||||
s = pd.to_numeric(df[col], errors="coerce")
|
||||
if s.notna().any():
|
||||
numeric_cols.append(col)
|
||||
return numeric_cols
|
||||
|
||||
@staticmethod
|
||||
def _compute_roc(
|
||||
y: np.ndarray, scores: np.ndarray
|
||||
) -> Tuple[np.ndarray, np.ndarray, np.ndarray, float]:
|
||||
fpr, tpr, thresholds = roc_curve(y, scores, pos_label=1)
|
||||
auc_val = float(auc(fpr, tpr))
|
||||
return fpr, tpr, thresholds, auc_val
|
||||
|
||||
@staticmethod
|
||||
def _best_threshold(
|
||||
fpr: np.ndarray, tpr: np.ndarray, thresholds: np.ndarray
|
||||
) -> Tuple[float, float, float]:
|
||||
youden = tpr - fpr
|
||||
idx = int(np.nanargmax(youden))
|
||||
return float(thresholds[idx]), float(tpr[idx]), float(fpr[idx])
|
||||
|
||||
@staticmethod
|
||||
def _plot_overlay(curves, title: str, out_path: Path) -> None:
|
||||
fig, ax = plt.subplots(figsize=(7, 5.5))
|
||||
cmap = plt.get_cmap("tab20")
|
||||
for i, (name, fpr, tpr, auc_val) in enumerate(curves):
|
||||
color = cmap(i % cmap.N)
|
||||
ax.plot(fpr, tpr, lw=1.6, color=color, label=f"{name} (AUC={auc_val:.3f})")
|
||||
ax.plot([0, 1], [0, 1], "k--", lw=1)
|
||||
ax.set_xlabel("False Positive Rate")
|
||||
ax.set_ylabel("True Positive Rate")
|
||||
ax.set_title(title)
|
||||
ax.legend(loc="upper left", fontsize="small")
|
||||
ax.grid(True, alpha=0.3, linestyle="--")
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=170)
|
||||
plt.close(fig)
|
||||
|
||||
@staticmethod
|
||||
def _plot_per_feature(
|
||||
fpr: np.ndarray, tpr: np.ndarray, auc_val: float, title: str, out_path: Path
|
||||
) -> None:
|
||||
fig, ax = plt.subplots(figsize=(5.5, 4.5))
|
||||
ax.plot(fpr, tpr, lw=1.8, label=f"AUC={auc_val:.3f}")
|
||||
ax.plot([0, 1], [0, 1], "k--", lw=1)
|
||||
ax.set_xlabel("False Positive Rate")
|
||||
ax.set_ylabel("True Positive Rate")
|
||||
ax.set_title(title)
|
||||
ax.legend(loc="lower right")
|
||||
ax.grid(True, alpha=0.3, linestyle="--")
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=170)
|
||||
plt.close(fig)
|
||||
|
||||
@staticmethod
|
||||
def _plot_roc_line(
|
||||
fpr: np.ndarray, tpr: np.ndarray, auc_val: float, title: str, out_path: Path
|
||||
) -> None:
|
||||
fig, ax = plt.subplots(figsize=(5.5, 4.5))
|
||||
ax.plot(fpr, tpr, lw=1.8, label=f"AUC={auc_val:.3f}")
|
||||
ax.plot([0, 1], [0, 1], "k--", lw=1)
|
||||
ax.set_xlabel("False Positive Rate")
|
||||
ax.set_ylabel("True Positive Rate")
|
||||
ax.set_title(title)
|
||||
ax.legend(loc="lower right")
|
||||
ax.grid(True, alpha=0.3, linestyle="--")
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=170)
|
||||
plt.close(fig)
|
||||
|
||||
def _oof_scores(
|
||||
self,
|
||||
model,
|
||||
X: np.ndarray,
|
||||
y: np.ndarray,
|
||||
n_splits: int,
|
||||
random_state: int,
|
||||
) -> Tuple[np.ndarray, np.ndarray]:
|
||||
skf = StratifiedKFold(
|
||||
n_splits=n_splits, shuffle=True, random_state=random_state
|
||||
)
|
||||
scores = np.zeros(len(y), dtype=float)
|
||||
for train_idx, test_idx in skf.split(X, y):
|
||||
X_train, X_test = X[train_idx], X[test_idx]
|
||||
y_train = y[train_idx]
|
||||
if np.unique(y_train).size < 2:
|
||||
continue
|
||||
fitted = clone(model)
|
||||
fitted.fit(X_train, y_train)
|
||||
if hasattr(fitted, "predict_proba"):
|
||||
fold_scores = fitted.predict_proba(X_test)[:, 1]
|
||||
elif hasattr(fitted, "decision_function"):
|
||||
fold_scores = fitted.decision_function(X_test)
|
||||
else:
|
||||
fold_scores = fitted.predict(X_test)
|
||||
scores[test_idx] = fold_scores
|
||||
return y.astype(int), scores
|
||||
|
||||
def _cv_roc_curves(
|
||||
self,
|
||||
model,
|
||||
X: np.ndarray,
|
||||
y: np.ndarray,
|
||||
n_splits: int,
|
||||
random_state: int,
|
||||
) -> List[Tuple[np.ndarray, np.ndarray, float]]:
|
||||
skf = StratifiedKFold(
|
||||
n_splits=n_splits, shuffle=True, random_state=random_state
|
||||
)
|
||||
curves = []
|
||||
for train_idx, test_idx in skf.split(X, y):
|
||||
X_train, X_test = X[train_idx], X[test_idx]
|
||||
y_train, y_test = y[train_idx], y[test_idx]
|
||||
if np.unique(y_train).size < 2 or np.unique(y_test).size < 2:
|
||||
continue
|
||||
fitted = clone(model)
|
||||
fitted.fit(X_train, y_train)
|
||||
if hasattr(fitted, "predict_proba"):
|
||||
scores = fitted.predict_proba(X_test)[:, 1]
|
||||
elif hasattr(fitted, "decision_function"):
|
||||
scores = fitted.decision_function(X_test)
|
||||
else:
|
||||
scores = fitted.predict(X_test)
|
||||
fpr, tpr, _ = roc_curve(y_test, scores, pos_label=1)
|
||||
auc_val = float(auc(fpr, tpr))
|
||||
curves.append((fpr, tpr, auc_val))
|
||||
return curves
|
||||
|
||||
@staticmethod
|
||||
def _plot_mean_roc(
|
||||
curves: List[Tuple[np.ndarray, np.ndarray, float]],
|
||||
title: str,
|
||||
out_path: Path,
|
||||
) -> None:
|
||||
if not curves:
|
||||
return
|
||||
mean_fpr = np.linspace(0.0, 1.0, 200)
|
||||
tprs = []
|
||||
aucs = []
|
||||
for fpr, tpr, auc_val in curves:
|
||||
tpr_interp = np.interp(mean_fpr, fpr, tpr)
|
||||
tpr_interp[0] = 0.0
|
||||
tprs.append(tpr_interp)
|
||||
aucs.append(auc_val)
|
||||
mean_tpr = np.mean(tprs, axis=0)
|
||||
mean_tpr[-1] = 1.0
|
||||
std_tpr = np.std(tprs, axis=0)
|
||||
mean_auc = float(np.mean(aucs))
|
||||
std_auc = float(np.std(aucs, ddof=0))
|
||||
|
||||
fig, ax = plt.subplots(figsize=(5.8, 4.6))
|
||||
ax.plot(mean_fpr, mean_tpr, lw=2, label=f"AUC={mean_auc:.3f}±{std_auc:.3f}")
|
||||
ax.fill_between(
|
||||
mean_fpr,
|
||||
np.maximum(mean_tpr - std_tpr, 0),
|
||||
np.minimum(mean_tpr + std_tpr, 1),
|
||||
color="grey",
|
||||
alpha=0.2,
|
||||
)
|
||||
ax.plot([0, 1], [0, 1], "k--", lw=1)
|
||||
ax.set_xlabel("False Positive Rate")
|
||||
ax.set_ylabel("True Positive Rate")
|
||||
ax.set_title(title)
|
||||
ax.legend(loc="lower right")
|
||||
ax.grid(True, alpha=0.3, linestyle="--")
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=170)
|
||||
plt.close(fig)
|
||||
|
||||
def _iter_categorical(self, df: pd.DataFrame, cols: List[str]):
|
||||
for col in cols:
|
||||
if col not in df.columns:
|
||||
continue
|
||||
s = df[col]
|
||||
vals = s.dropna().unique().tolist()
|
||||
try:
|
||||
vals = sorted(vals)
|
||||
except Exception:
|
||||
pass
|
||||
for v in vals:
|
||||
name = f"{col}=={v}"
|
||||
ind = (s == v).astype(int)
|
||||
yield name, ind
|
||||
|
||||
def _feature_matrix(
|
||||
self, df: pd.DataFrame, include_categorical: bool
|
||||
) -> Tuple[np.ndarray, np.ndarray, List[str]]:
|
||||
numeric_cols = self._numeric_columns(df)
|
||||
X_num = df[numeric_cols].apply(pd.to_numeric, errors="coerce")
|
||||
for col in numeric_cols:
|
||||
med = pd.to_numeric(X_num[col], errors="coerce").median()
|
||||
X_num[col] = pd.to_numeric(X_num[col], errors="coerce").fillna(med)
|
||||
|
||||
parts = [X_num]
|
||||
feat_names = list(X_num.columns)
|
||||
|
||||
if include_categorical and self.cat_cols:
|
||||
cat_cols = [c for c in self.cat_cols if c in df.columns]
|
||||
if cat_cols:
|
||||
df_cats = pd.get_dummies(
|
||||
df[cat_cols].astype("category"), drop_first=False, prefix=cat_cols
|
||||
)
|
||||
parts.append(df_cats)
|
||||
feat_names.extend(list(df_cats.columns))
|
||||
|
||||
X = pd.concat(parts, axis=1).values.astype(np.float32)
|
||||
labels = df[self.label_col]
|
||||
y_all, keep_mask = self._select_binary_labels(labels)
|
||||
y = y_all[keep_mask]
|
||||
X = X[keep_mask]
|
||||
return X, y.astype(int), feat_names
|
||||
|
||||
def univariate_roc(
|
||||
self, merge: bool = False, include_categorical: bool = False
|
||||
) -> pd.DataFrame:
|
||||
clinical = self._build_clinical()
|
||||
df = clinical.df.copy()
|
||||
labels = df[self.label_col]
|
||||
y_all, keep_mask = self._select_binary_labels(labels)
|
||||
|
||||
if self.debug:
|
||||
for col in ("IOP_raw", "IOP_corr"):
|
||||
if col not in df.columns:
|
||||
print(f"[debug] {col} missing from df")
|
||||
continue
|
||||
s = pd.to_numeric(df[col], errors="coerce")
|
||||
print(
|
||||
f"[debug] {col}: non-null={int(s.notna().sum())}, unique={int(s.nunique(dropna=True))}"
|
||||
)
|
||||
|
||||
plot_dir = self.output_dir / "papila_univariate_roc" / "plots"
|
||||
plot_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
rows = []
|
||||
curves = []
|
||||
|
||||
numeric_cols = self._numeric_columns(df)
|
||||
for col in numeric_cols:
|
||||
series = pd.to_numeric(df[col], errors="coerce")
|
||||
mask = keep_mask & series.notna().values
|
||||
y = y_all[mask]
|
||||
scores = series.values[mask].astype(float)
|
||||
if y.size < 2 or np.unique(y).size < 2:
|
||||
continue
|
||||
if np.nanmin(scores) == np.nanmax(scores):
|
||||
continue
|
||||
fpr, tpr, thresholds, auc_val = self._compute_roc(y, scores)
|
||||
thr, best_tpr, best_fpr = self._best_threshold(fpr, tpr, thresholds)
|
||||
direction = "high" if auc_val >= 0.5 else "low"
|
||||
title = f"{col} (n={y.size}, direction={direction})"
|
||||
if merge:
|
||||
out_path = plot_dir / f"roc_{self._sanitize(col)}.png"
|
||||
self._plot_per_feature(fpr, tpr, auc_val, title, out_path)
|
||||
curves.append((col, fpr, tpr, auc_val))
|
||||
rows.append(
|
||||
{
|
||||
"feature": col,
|
||||
"kind": "numeric",
|
||||
"n": int(y.size),
|
||||
"auc": auc_val,
|
||||
"direction": direction,
|
||||
"best_threshold": thr,
|
||||
"best_tpr": best_tpr,
|
||||
"best_fpr": best_fpr,
|
||||
"best_specificity": 1.0 - best_fpr,
|
||||
}
|
||||
)
|
||||
|
||||
if include_categorical:
|
||||
cat_cols_use = [c for c in self.cat_cols if c not in self.exclude_cols]
|
||||
for name, ind in self._iter_categorical(df, cat_cols_use):
|
||||
mask = keep_mask & ind.notna().values
|
||||
y = y_all[mask]
|
||||
scores = ind.values[mask].astype(float)
|
||||
if y.size < 2 or np.unique(y).size < 2:
|
||||
continue
|
||||
if np.nanmin(scores) == np.nanmax(scores):
|
||||
continue
|
||||
fpr, tpr, thresholds, auc_val = self._compute_roc(y, scores)
|
||||
thr, best_tpr, best_fpr = self._best_threshold(fpr, tpr, thresholds)
|
||||
direction = "high" if auc_val >= 0.5 else "low"
|
||||
title = f"{name} (n={y.size}, direction={direction})"
|
||||
if merge:
|
||||
out_path = plot_dir / f"roc_{self._sanitize(name)}.png"
|
||||
self._plot_per_feature(fpr, tpr, auc_val, title, out_path)
|
||||
curves.append((name, fpr, tpr, auc_val))
|
||||
rows.append(
|
||||
{
|
||||
"feature": name,
|
||||
"kind": "categorical",
|
||||
"n": int(y.size),
|
||||
"auc": auc_val,
|
||||
"direction": direction,
|
||||
"best_threshold": thr,
|
||||
"best_tpr": best_tpr,
|
||||
"best_fpr": best_fpr,
|
||||
"best_specificity": 1.0 - best_fpr,
|
||||
}
|
||||
)
|
||||
|
||||
if not rows:
|
||||
raise SystemExit(
|
||||
"No valid features produced ROC curves. Check labels and feature columns."
|
||||
)
|
||||
|
||||
overlay_path = plot_dir / "roc_overlay.png"
|
||||
if not merge:
|
||||
self._plot_overlay(curves, "Univariate ROC curves", overlay_path)
|
||||
|
||||
out_df = pd.DataFrame(rows).sort_values(by="auc", ascending=False)
|
||||
out_csv = self.output_dir / "papila_univariate_roc" / "summary.csv"
|
||||
out_csv.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_df.to_csv(out_csv, index=False)
|
||||
return out_df
|
||||
|
||||
def random_forest(
|
||||
self,
|
||||
include_categorical: bool = True,
|
||||
n_estimators: int = 500,
|
||||
max_depth: int | None = None,
|
||||
min_samples_leaf: int = 1,
|
||||
max_features: str | None = "sqrt",
|
||||
class_weight: str | None = "balanced",
|
||||
max_samples: float | None = None,
|
||||
random_state: int = 42,
|
||||
top_n: int = 25,
|
||||
n_splits: int = 5,
|
||||
drop_missing: bool = False,
|
||||
nerf: bool = False,
|
||||
drop_age: bool = False,
|
||||
) -> pd.DataFrame:
|
||||
clinical = self._build_clinical()
|
||||
df = clinical.df.copy()
|
||||
original_exclude = set(self.exclude_cols)
|
||||
if nerf:
|
||||
self.exclude_cols = set(self.exclude_cols)
|
||||
drop_missing = True
|
||||
if drop_age:
|
||||
self.exclude_cols = set(self.exclude_cols) | {"Age"}
|
||||
if drop_missing:
|
||||
numeric_cols = self._numeric_columns(df)
|
||||
df = df.dropna(subset=numeric_cols)
|
||||
X, y, feat_names = self._feature_matrix(
|
||||
df, include_categorical=include_categorical
|
||||
)
|
||||
|
||||
if X.size == 0 or np.unique(y).size < 2:
|
||||
raise SystemExit(
|
||||
"Not enough data after filtering labels for Random Forest."
|
||||
)
|
||||
|
||||
if nerf:
|
||||
n_estimators = 200
|
||||
max_depth = 5
|
||||
min_samples_leaf = 5
|
||||
max_features = "sqrt"
|
||||
class_weight = None
|
||||
max_samples = 0.7
|
||||
self.exclude_cols = original_exclude
|
||||
|
||||
clf = RandomForestClassifier(
|
||||
n_estimators=n_estimators,
|
||||
max_depth=max_depth,
|
||||
min_samples_leaf=min_samples_leaf,
|
||||
max_features=max_features,
|
||||
class_weight=class_weight,
|
||||
max_samples=max_samples,
|
||||
random_state=random_state,
|
||||
n_jobs=-1,
|
||||
)
|
||||
clf.fit(X, y)
|
||||
importances = clf.feature_importances_.astype(float)
|
||||
|
||||
rows = []
|
||||
for name, val in zip(feat_names, importances):
|
||||
rows.append({"feature": name, "importance": float(val)})
|
||||
|
||||
out_df = pd.DataFrame(rows).sort_values(by="importance", ascending=False)
|
||||
out_dir = self.output_dir / (
|
||||
"papila_random_forest_nerfed" if nerf else "papila_random_forest"
|
||||
)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_df.to_csv(out_dir / "feature_importance.csv", index=False)
|
||||
|
||||
top_df = out_df.head(top_n)
|
||||
fig, ax = plt.subplots(figsize=(7, 6))
|
||||
ax.barh(top_df["feature"], top_df["importance"], color="steelblue")
|
||||
ax.invert_yaxis()
|
||||
ax.set_xlabel("Importance (Gini)")
|
||||
ax.set_title(f"Random Forest Feature Importance")
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_dir / "feature_importance_top.png", dpi=170)
|
||||
plt.close(fig)
|
||||
if self.debug:
|
||||
age_rows = out_df[out_df["feature"] == "Age"]
|
||||
if not age_rows.empty:
|
||||
age_imp = float(age_rows["importance"].iloc[0])
|
||||
print(f"[debug] RF importance Age = {age_imp:.4f}")
|
||||
|
||||
y_oof, scores_oof = self._oof_scores(clf, X, y, n_splits, random_state)
|
||||
fpr, tpr, _, auc_val = self._compute_roc(y_oof, scores_oof)
|
||||
curves = self._cv_roc_curves(clf, X, y, n_splits, random_state)
|
||||
self._plot_mean_roc(
|
||||
curves,
|
||||
"Random Forest ROC (mean ± SD)",
|
||||
out_dir / "roc_mean.png",
|
||||
)
|
||||
|
||||
return out_df
|
||||
|
||||
def _cv_binary_metrics(
|
||||
self,
|
||||
model,
|
||||
X: np.ndarray,
|
||||
y: np.ndarray,
|
||||
n_splits: int,
|
||||
random_state: int,
|
||||
) -> pd.DataFrame:
|
||||
skf = StratifiedKFold(
|
||||
n_splits=n_splits, shuffle=True, random_state=random_state
|
||||
)
|
||||
rows = []
|
||||
for fold, (train_idx, test_idx) in enumerate(skf.split(X, y), start=1):
|
||||
X_train, X_test = X[train_idx], X[test_idx]
|
||||
y_train, y_test = y[train_idx], y[test_idx]
|
||||
if np.unique(y_train).size < 2 or np.unique(y_test).size < 2:
|
||||
continue
|
||||
model.fit(X_train, y_train)
|
||||
if hasattr(model, "predict_proba"):
|
||||
scores = model.predict_proba(X_test)[:, 1]
|
||||
elif hasattr(model, "decision_function"):
|
||||
scores = model.decision_function(X_test)
|
||||
else:
|
||||
scores = model.predict(X_test)
|
||||
preds = model.predict(X_test)
|
||||
auc_val = float(roc_auc_score(y_test, scores))
|
||||
acc_val = float(accuracy_score(y_test, preds))
|
||||
rows.append(
|
||||
{
|
||||
"fold": int(fold),
|
||||
"n": int(len(y_test)),
|
||||
"auc": auc_val,
|
||||
"acc": acc_val,
|
||||
}
|
||||
)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
def svm(
|
||||
self,
|
||||
include_categorical: bool = True,
|
||||
kernel: str = "rbf",
|
||||
C: float = 1.0,
|
||||
gamma: str = "scale",
|
||||
n_splits: int = 5,
|
||||
random_state: int = 42,
|
||||
) -> pd.DataFrame:
|
||||
clinical = self._build_clinical()
|
||||
df = clinical.df.copy()
|
||||
X, y, feat_names = self._feature_matrix(
|
||||
df, include_categorical=include_categorical
|
||||
)
|
||||
if X.size == 0 or np.unique(y).size < 2:
|
||||
raise SystemExit("Not enough data after filtering labels for SVM.")
|
||||
|
||||
model = Pipeline(
|
||||
[
|
||||
("scale", StandardScaler()),
|
||||
(
|
||||
"svm",
|
||||
SVC(
|
||||
kernel=kernel,
|
||||
C=C,
|
||||
gamma=gamma,
|
||||
probability=True,
|
||||
class_weight="balanced",
|
||||
random_state=random_state,
|
||||
),
|
||||
),
|
||||
]
|
||||
)
|
||||
fold_df = self._cv_binary_metrics(model, X, y, n_splits, random_state)
|
||||
if fold_df.empty:
|
||||
raise SystemExit("SVM produced no valid folds (check class balance).")
|
||||
|
||||
y_oof, scores_oof = self._oof_scores(model, X, y, n_splits, random_state)
|
||||
fpr, tpr, _, auc_val = self._compute_roc(y_oof, scores_oof)
|
||||
curves = self._cv_roc_curves(model, X, y, n_splits, random_state)
|
||||
|
||||
summary = pd.DataFrame(
|
||||
[
|
||||
{
|
||||
"metric": "auc",
|
||||
"mean": float(fold_df["auc"].mean()),
|
||||
"std": float(fold_df["auc"].std(ddof=0)),
|
||||
"oof_auc": float(auc_val),
|
||||
},
|
||||
{
|
||||
"metric": "acc",
|
||||
"mean": float(fold_df["acc"].mean()),
|
||||
"std": float(fold_df["acc"].std(ddof=0)),
|
||||
},
|
||||
]
|
||||
)
|
||||
out_dir = self.output_dir / "papila_svm"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
fold_df.to_csv(out_dir / "fold_metrics.csv", index=False)
|
||||
summary.to_csv(out_dir / "summary.csv", index=False)
|
||||
self._plot_mean_roc(
|
||||
curves,
|
||||
"SVM ROC (mean ± SD)",
|
||||
out_dir / "roc_mean.png",
|
||||
)
|
||||
return fold_df
|
||||
|
||||
def knn(
|
||||
self,
|
||||
include_categorical: bool = True,
|
||||
n_neighbors: int = 5,
|
||||
weights: str = "distance",
|
||||
n_splits: int = 5,
|
||||
random_state: int = 42,
|
||||
) -> pd.DataFrame:
|
||||
clinical = self._build_clinical()
|
||||
df = clinical.df.copy()
|
||||
X, y, feat_names = self._feature_matrix(
|
||||
df, include_categorical=include_categorical
|
||||
)
|
||||
if X.size == 0 or np.unique(y).size < 2:
|
||||
raise SystemExit("Not enough data after filtering labels for KNN.")
|
||||
|
||||
model = Pipeline(
|
||||
[
|
||||
("scale", StandardScaler()),
|
||||
("knn", KNeighborsClassifier(n_neighbors=n_neighbors, weights=weights)),
|
||||
]
|
||||
)
|
||||
fold_df = self._cv_binary_metrics(model, X, y, n_splits, random_state)
|
||||
if fold_df.empty:
|
||||
raise SystemExit("KNN produced no valid folds (check class balance).")
|
||||
|
||||
y_oof, scores_oof = self._oof_scores(model, X, y, n_splits, random_state)
|
||||
fpr, tpr, _, auc_val = self._compute_roc(y_oof, scores_oof)
|
||||
curves = self._cv_roc_curves(model, X, y, n_splits, random_state)
|
||||
|
||||
summary = pd.DataFrame(
|
||||
[
|
||||
{
|
||||
"metric": "auc",
|
||||
"mean": float(fold_df["auc"].mean()),
|
||||
"std": float(fold_df["auc"].std(ddof=0)),
|
||||
"oof_auc": float(auc_val),
|
||||
},
|
||||
{
|
||||
"metric": "acc",
|
||||
"mean": float(fold_df["acc"].mean()),
|
||||
"std": float(fold_df["acc"].std(ddof=0)),
|
||||
},
|
||||
]
|
||||
)
|
||||
out_dir = self.output_dir / "papila_knn"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
fold_df.to_csv(out_dir / "fold_metrics.csv", index=False)
|
||||
summary.to_csv(out_dir / "summary.csv", index=False)
|
||||
self._plot_mean_roc(
|
||||
curves,
|
||||
"KNN ROC (mean ± SD)",
|
||||
out_dir / "roc_mean.png",
|
||||
)
|
||||
return fold_df
|
||||
|
||||
def logistic_regression(
|
||||
self,
|
||||
include_categorical: bool = True,
|
||||
C: float = 1.0,
|
||||
max_iter: int = 1000,
|
||||
n_splits: int = 5,
|
||||
random_state: int = 42,
|
||||
drop_age: bool = False,
|
||||
nerf: bool = False,
|
||||
) -> pd.DataFrame:
|
||||
clinical = self._build_clinical()
|
||||
df = clinical.df.copy()
|
||||
original_exclude = set(self.exclude_cols)
|
||||
if drop_age:
|
||||
self.exclude_cols = set(self.exclude_cols) | {"Age"}
|
||||
X, y, feat_names = self._feature_matrix(
|
||||
df, include_categorical=include_categorical
|
||||
)
|
||||
self.exclude_cols = original_exclude
|
||||
if X.size == 0 or np.unique(y).size < 2:
|
||||
raise SystemExit(
|
||||
"Not enough data after filtering labels for Logistic Regression."
|
||||
)
|
||||
|
||||
class_weight = "balanced"
|
||||
penalty = "l2"
|
||||
solver = "lbfgs"
|
||||
if nerf:
|
||||
C = 0.05
|
||||
class_weight = None
|
||||
penalty = "l1"
|
||||
solver = "liblinear"
|
||||
|
||||
model = Pipeline(
|
||||
[
|
||||
("scale", StandardScaler()),
|
||||
(
|
||||
"logreg",
|
||||
LogisticRegression(
|
||||
C=C,
|
||||
max_iter=max_iter,
|
||||
class_weight=class_weight,
|
||||
penalty=penalty,
|
||||
solver=solver,
|
||||
),
|
||||
),
|
||||
]
|
||||
)
|
||||
fold_df = self._cv_binary_metrics(model, X, y, n_splits, random_state)
|
||||
if fold_df.empty:
|
||||
raise SystemExit(
|
||||
"Logistic Regression produced no valid folds (check class balance)."
|
||||
)
|
||||
|
||||
y_oof, scores_oof = self._oof_scores(model, X, y, n_splits, random_state)
|
||||
fpr, tpr, _, auc_val = self._compute_roc(y_oof, scores_oof)
|
||||
curves = self._cv_roc_curves(model, X, y, n_splits, random_state)
|
||||
|
||||
summary = pd.DataFrame(
|
||||
[
|
||||
{
|
||||
"metric": "auc",
|
||||
"mean": float(fold_df["auc"].mean()),
|
||||
"std": float(fold_df["auc"].std(ddof=0)),
|
||||
"oof_auc": float(auc_val),
|
||||
},
|
||||
{
|
||||
"metric": "acc",
|
||||
"mean": float(fold_df["acc"].mean()),
|
||||
"std": float(fold_df["acc"].std(ddof=0)),
|
||||
},
|
||||
]
|
||||
)
|
||||
out_dir = self.output_dir / "papila_logistic_regression"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
fold_df.to_csv(out_dir / "fold_metrics.csv", index=False)
|
||||
summary.to_csv(out_dir / "summary.csv", index=False)
|
||||
self._plot_mean_roc(
|
||||
curves,
|
||||
"Logistic Regression ROC (mean ± SD)",
|
||||
out_dir / "roc_mean.png",
|
||||
)
|
||||
return fold_df
|
||||
|
||||
|
||||
ba = basic_analytics()
|
||||
roc_df = ba.univariate_roc(merge=False, include_categorical=False)
|
||||
rf_df = ba.random_forest(include_categorical=True, nerf=False)
|
||||
svm_df = ba.svm(include_categorical=True)
|
||||
knn_df = ba.knn(include_categorical=True)
|
||||
lr_df = ba.logistic_regression(include_categorical=True, nerf=True)
|
||||
clinical = ba._build_clinical()
|
||||
clinical.df["Diagnosis"].value_counts()
|
||||
Executable
+327
@@ -0,0 +1,327 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Train a CNN (resnet50 backbone), extract logits, and train RF on logits+metadata with 5-fold CV."""
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from PIL import Image
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.utils.data import DataLoader, Dataset
|
||||
from torchvision import transforms
|
||||
from sklearn.ensemble import RandomForestClassifier
|
||||
from sklearn.metrics import accuracy_score, roc_auc_score
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from classes import build_papila_clinical
|
||||
from classes.backbones import BACKBONES, load_backbone_weights
|
||||
|
||||
|
||||
# ---------------------------
|
||||
# Config (edit in IDE)
|
||||
# ---------------------------
|
||||
IMAGE_DIR = "Papila/FundusImages"
|
||||
CLINICAL_DIR = "Papila/ClinicalData"
|
||||
LABEL_COL = "Diagnosis"
|
||||
CAT_COLS = ["Gender", "Phakic/Pseudophakic"]
|
||||
EVAL_MODE = "binary" # "binary" or "multiclass"
|
||||
N_SPLITS = 5
|
||||
FOLD_SEED = 42
|
||||
HOLDOUT_SEED = 123
|
||||
HOLDOUT_PATIENTS_PER_CLASS = 6
|
||||
|
||||
BACKBONE_NAME = "resnet50"
|
||||
BATCH_SIZE = 8
|
||||
EPOCHS = 40
|
||||
LR = 1e-4
|
||||
WEIGHT_DECAY = 1e-5
|
||||
|
||||
RF_TREES = 500
|
||||
RF_MAX_DEPTH = None
|
||||
RF_MIN_SAMPLES_LEAF = 1
|
||||
|
||||
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
OUTPUT_DIR = Path("analysis_data/basic_analysis/cnn_logits_rf_cv")
|
||||
PRINT_EPOCH_REPORT = True
|
||||
EPOCH_REPORT_EVERY = 1
|
||||
|
||||
|
||||
class PapilaImageDataset(Dataset):
|
||||
def __init__(
|
||||
self, clinical, df: pd.DataFrame, label_col: str, img_transform
|
||||
) -> None:
|
||||
self.clinical = clinical
|
||||
self.df = df.reset_index(drop=True)
|
||||
self.label_col = label_col
|
||||
self.img_transform = img_transform
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.df)
|
||||
|
||||
def __getitem__(self, idx: int):
|
||||
row = self.df.iloc[idx]
|
||||
img_path = self.clinical.get_image_path(row)
|
||||
image = Image.open(img_path).convert("RGB")
|
||||
x_img = self.img_transform(image)
|
||||
y = int(row[self.label_col])
|
||||
x_md = self.clinical.vectorize_row(row).astype(np.float32)
|
||||
return x_img, y, x_md
|
||||
|
||||
|
||||
class CNNHead(nn.Module):
|
||||
def __init__(self, backbone_name: str, num_classes: int) -> None:
|
||||
super().__init__()
|
||||
spec = BACKBONES[backbone_name]
|
||||
backbone = spec.ctor(weights=spec.weights_default)
|
||||
if backbone_name.startswith("refuge"):
|
||||
load_backbone_weights(backbone_name, backbone)
|
||||
out_dim, backbone = spec.strip(backbone)
|
||||
self.backbone = backbone
|
||||
self.head = nn.Linear(out_dim, num_classes)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
feats = self.backbone(x)
|
||||
return self.head(feats)
|
||||
|
||||
|
||||
def _set_seed(seed: int) -> None:
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
|
||||
|
||||
def _auc_score(y_true: np.ndarray, probs: np.ndarray, num_classes: int) -> float:
|
||||
try:
|
||||
if num_classes == 2:
|
||||
return float(roc_auc_score(y_true, probs[:, 1]))
|
||||
return float(roc_auc_score(y_true, probs, multi_class="ovr", average="macro"))
|
||||
except Exception:
|
||||
return float("nan")
|
||||
|
||||
|
||||
def _prepare_clinical() -> Tuple[object, pd.DataFrame]:
|
||||
clinical = build_papila_clinical(
|
||||
image_dir=IMAGE_DIR,
|
||||
clinical_dir=CLINICAL_DIR,
|
||||
label_col=LABEL_COL,
|
||||
cat_cols=CAT_COLS,
|
||||
n_splits=N_SPLITS,
|
||||
random_seed=FOLD_SEED,
|
||||
)
|
||||
df = clinical.df.copy()
|
||||
if EVAL_MODE == "binary":
|
||||
df = df[df[LABEL_COL].isin([0, 1])].reset_index(drop=True)
|
||||
return clinical, df
|
||||
|
||||
|
||||
def _split_holdout_by_patient(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:
|
||||
rng = np.random.default_rng(HOLDOUT_SEED)
|
||||
patient_label = (
|
||||
df.groupby("Patient ID")[LABEL_COL]
|
||||
.agg(lambda s: int(s.mode().iloc[0]))
|
||||
.reset_index()
|
||||
)
|
||||
holdout_patients = []
|
||||
for lbl, grp in patient_label.groupby(LABEL_COL):
|
||||
candidates = grp["Patient ID"].to_numpy()
|
||||
n = min(HOLDOUT_PATIENTS_PER_CLASS, len(candidates))
|
||||
if n <= 0:
|
||||
continue
|
||||
selected = rng.choice(candidates, size=n, replace=False)
|
||||
holdout_patients.extend(selected.tolist())
|
||||
holdout_patients = sorted(set(holdout_patients))
|
||||
holdout_df = df[df["Patient ID"].isin(holdout_patients)].reset_index(drop=True)
|
||||
train_df = df[~df["Patient ID"].isin(holdout_patients)].reset_index(drop=True)
|
||||
return train_df, holdout_df
|
||||
|
||||
|
||||
def _rebuild_clinical_from_df(clinical, df: pd.DataFrame) -> object:
|
||||
clinical.frames = [df.copy()]
|
||||
clinical.df = df.copy()
|
||||
clinical._infer_or_validate_feature_types()
|
||||
clinical._compute_numeric_stats()
|
||||
clinical._build_cat_maps()
|
||||
clinical._compute_feature_dim()
|
||||
clinical._build_kfold_indices()
|
||||
return clinical
|
||||
|
||||
|
||||
def _train_cnn(
|
||||
model: nn.Module, loader: DataLoader, num_classes: int, fold: int
|
||||
) -> None:
|
||||
model.train()
|
||||
optimizer = torch.optim.Adam(model.parameters(), lr=LR, weight_decay=WEIGHT_DECAY)
|
||||
criterion = nn.CrossEntropyLoss()
|
||||
for epoch in range(EPOCHS):
|
||||
running_loss = 0.0
|
||||
correct = 0
|
||||
total = 0
|
||||
for x_img, y, _x_md in loader:
|
||||
x_img = x_img.to(DEVICE)
|
||||
y = y.to(DEVICE)
|
||||
optimizer.zero_grad()
|
||||
logits = model(x_img)
|
||||
loss = criterion(logits, y)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
running_loss += float(loss.item()) * int(y.size(0))
|
||||
pred = torch.argmax(logits, dim=1)
|
||||
correct += int((pred == y).sum().item())
|
||||
total += int(y.size(0))
|
||||
|
||||
if PRINT_EPOCH_REPORT and ((epoch + 1) % EPOCH_REPORT_EVERY == 0):
|
||||
avg_loss = running_loss / max(total, 1)
|
||||
train_acc = correct / max(total, 1)
|
||||
print(
|
||||
f"[fold {fold + 1}/{N_SPLITS}] epoch {epoch + 1}/{EPOCHS} "
|
||||
f"train_loss={avg_loss:.4f} train_acc={train_acc:.4f}"
|
||||
)
|
||||
|
||||
|
||||
def _infer_logits(
|
||||
model: nn.Module, loader: DataLoader
|
||||
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
model.eval()
|
||||
logits_all, probs_all, y_all, md_all = [], [], [], []
|
||||
with torch.no_grad():
|
||||
for x_img, y, x_md in loader:
|
||||
x_img = x_img.to(DEVICE)
|
||||
logits = model(x_img).cpu().numpy()
|
||||
probs = torch.softmax(torch.from_numpy(logits), dim=1).numpy()
|
||||
logits_all.append(logits)
|
||||
probs_all.append(probs)
|
||||
y_all.append(y.numpy())
|
||||
md_all.append(x_md.numpy())
|
||||
return (
|
||||
np.concatenate(y_all, axis=0),
|
||||
np.concatenate(logits_all, axis=0),
|
||||
np.concatenate(md_all, axis=0),
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
_set_seed(FOLD_SEED)
|
||||
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
num_classes = 2 if EVAL_MODE == "binary" else 3
|
||||
train_tf = transforms.Compose(
|
||||
[
|
||||
transforms.Resize((224, 224)),
|
||||
transforms.RandomHorizontalFlip(),
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
|
||||
]
|
||||
)
|
||||
eval_tf = transforms.Compose(
|
||||
[
|
||||
transforms.Resize((224, 224)),
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
|
||||
]
|
||||
)
|
||||
|
||||
clinical, df = _prepare_clinical()
|
||||
train_df, holdout_df = _split_holdout_by_patient(df)
|
||||
clinical = _rebuild_clinical_from_df(clinical, train_df)
|
||||
holdout_df.to_csv(OUTPUT_DIR / "holdout_patients.csv", index=False)
|
||||
|
||||
rows: List[Dict[str, object]] = []
|
||||
holdout_rows: List[Dict[str, object]] = []
|
||||
|
||||
for fold in range(N_SPLITS):
|
||||
print(f"\n[info] Starting fold {fold + 1}/{N_SPLITS}")
|
||||
fold_train_df, fold_val_df = clinical.get_split_dfs(fold)
|
||||
ds_train = PapilaImageDataset(clinical, fold_train_df, LABEL_COL, train_tf)
|
||||
ds_val = PapilaImageDataset(clinical, fold_val_df, LABEL_COL, eval_tf)
|
||||
ds_holdout = PapilaImageDataset(clinical, holdout_df, LABEL_COL, eval_tf)
|
||||
|
||||
dl_train = DataLoader(
|
||||
ds_train, batch_size=BATCH_SIZE, shuffle=True, num_workers=0
|
||||
)
|
||||
dl_val = DataLoader(ds_val, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)
|
||||
dl_holdout = DataLoader(
|
||||
ds_holdout, batch_size=BATCH_SIZE, shuffle=False, num_workers=0
|
||||
)
|
||||
|
||||
model = CNNHead(BACKBONE_NAME, num_classes=num_classes).to(DEVICE)
|
||||
_train_cnn(model, dl_train, num_classes=num_classes, fold=fold)
|
||||
|
||||
y_tr, log_tr, md_tr = _infer_logits(
|
||||
model,
|
||||
DataLoader(ds_train, batch_size=BATCH_SIZE, shuffle=False, num_workers=0),
|
||||
)
|
||||
y_va, log_va, md_va = _infer_logits(model, dl_val)
|
||||
y_ho, log_ho, md_ho = _infer_logits(model, dl_holdout)
|
||||
|
||||
np.save(OUTPUT_DIR / f"fold{fold}_train_logits.npy", log_tr)
|
||||
np.save(OUTPUT_DIR / f"fold{fold}_val_logits.npy", log_va)
|
||||
np.save(OUTPUT_DIR / f"fold{fold}_holdout_logits.npy", log_ho)
|
||||
|
||||
X_tr = np.concatenate([log_tr, md_tr], axis=1)
|
||||
X_va = np.concatenate([log_va, md_va], axis=1)
|
||||
X_ho = np.concatenate([log_ho, md_ho], axis=1)
|
||||
|
||||
rf = RandomForestClassifier(
|
||||
n_estimators=RF_TREES,
|
||||
max_depth=RF_MAX_DEPTH,
|
||||
min_samples_leaf=RF_MIN_SAMPLES_LEAF,
|
||||
class_weight="balanced",
|
||||
random_state=FOLD_SEED + fold,
|
||||
n_jobs=-1,
|
||||
)
|
||||
rf.fit(X_tr, y_tr)
|
||||
|
||||
p_va = rf.predict_proba(X_va)
|
||||
p_ho = rf.predict_proba(X_ho)
|
||||
pred_va = np.argmax(p_va, axis=1)
|
||||
pred_ho = np.argmax(p_ho, axis=1)
|
||||
|
||||
rows.append(
|
||||
{
|
||||
"fold": fold,
|
||||
"val_acc": float(accuracy_score(y_va, pred_va)),
|
||||
"val_auc": _auc_score(y_va, p_va, num_classes),
|
||||
"n_val": int(len(y_va)),
|
||||
}
|
||||
)
|
||||
holdout_rows.append(
|
||||
{
|
||||
"fold": fold,
|
||||
"holdout_acc": float(accuracy_score(y_ho, pred_ho)),
|
||||
"holdout_auc": _auc_score(y_ho, p_ho, num_classes),
|
||||
"n_holdout": int(len(y_ho)),
|
||||
}
|
||||
)
|
||||
print(
|
||||
f"[info] Fold {fold + 1} RF: val_acc={rows[-1]['val_acc']:.4f} val_auc={rows[-1]['val_auc']:.4f} "
|
||||
f"| holdout_acc={holdout_rows[-1]['holdout_acc']:.4f} holdout_auc={holdout_rows[-1]['holdout_auc']:.4f}"
|
||||
)
|
||||
|
||||
fold_df = pd.DataFrame(rows)
|
||||
holdout_df = pd.DataFrame(holdout_rows)
|
||||
fold_df.to_csv(OUTPUT_DIR / "rf_val_metrics.csv", index=False)
|
||||
holdout_df.to_csv(OUTPUT_DIR / "rf_holdout_metrics.csv", index=False)
|
||||
|
||||
print("\nRF validation metrics:")
|
||||
print(fold_df.to_string(index=False, float_format=lambda x: f"{x:.4f}"))
|
||||
print("\nRF holdout metrics:")
|
||||
print(holdout_df.to_string(index=False, float_format=lambda x: f"{x:.4f}"))
|
||||
print(
|
||||
f"\nMeans: val_acc={fold_df['val_acc'].mean():.4f}, val_auc={fold_df['val_auc'].mean():.4f}, "
|
||||
f"holdout_acc={holdout_df['holdout_acc'].mean():.4f}, holdout_auc={holdout_df['holdout_auc'].mean():.4f}"
|
||||
)
|
||||
print(f"\nSaved outputs to: {OUTPUT_DIR}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Thin CLI wrapper that runs V2 hypertower modes sequentially."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import argparse
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from classes.v2.v2_hypertower import build_parser, run_mode
|
||||
|
||||
|
||||
def parse_args():
|
||||
ap = argparse.ArgumentParser(
|
||||
description="Run selected eval/tower mode combinations sequentially."
|
||||
)
|
||||
ap.add_argument(
|
||||
"--eval-modes",
|
||||
nargs="+",
|
||||
choices=["binary", "multiclass"],
|
||||
default=["binary", "multiclass"],
|
||||
)
|
||||
ap.add_argument(
|
||||
"--tower-modes",
|
||||
nargs="+",
|
||||
choices=["single", "ensemble", "bilateral", "classic"],
|
||||
default=["single", "ensemble", "bilateral"],
|
||||
)
|
||||
return ap.parse_known_args()
|
||||
|
||||
|
||||
def main():
|
||||
seq_args, remaining = parse_args()
|
||||
base_parser = build_parser()
|
||||
for eval_mode in seq_args.eval_modes:
|
||||
for tower_mode in seq_args.tower_modes:
|
||||
tower_mode = "single" if tower_mode == "classic" else tower_mode
|
||||
cli = list(remaining) + ["--eval-mode", eval_mode, "--tower-mode", tower_mode]
|
||||
args = base_parser.parse_args(cli)
|
||||
run_mode(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,725 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Compare single-eye OD baseline vs SiameseImageTower bilateral model.
|
||||
|
||||
Key differences from compare_dual_eye_towers.py:
|
||||
- Uses SiameseImageTower (shared backbone, f_mean + f_delta output).
|
||||
- Reports BEST-epoch val metrics per fold (not final-epoch), with the
|
||||
corresponding holdout metrics snapped at the same checkpoint.
|
||||
- Both models are always evaluated on patient-level samples (matched n).
|
||||
- Optionally includes two-single-merge as a second reference point.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import csv
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from sklearn.metrics import roc_auc_score
|
||||
from torch import nn
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from classes.v2 import (
|
||||
ImageTower,
|
||||
PatientFirstSplitManager,
|
||||
SiameseImageTower,
|
||||
SlotDataset,
|
||||
build_papila_data,
|
||||
build_papila_profile,
|
||||
slot_collate,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reproducibility
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def seed_everything(seed: int) -> None:
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class BaselineCNN(nn.Module):
|
||||
"""Single-eye (OD) image tower with a linear head."""
|
||||
|
||||
def __init__(self, *, backbone: str, freeze_ratio: float, num_classes: int, augment: bool):
|
||||
super().__init__()
|
||||
self.tower = ImageTower(
|
||||
backbone=backbone,
|
||||
freeze_ratio=freeze_ratio,
|
||||
augment=augment,
|
||||
use_se=False,
|
||||
)
|
||||
self.head = nn.Linear(self.tower.out_dim, num_classes)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.head(self.tower(x))
|
||||
|
||||
|
||||
class SiameseCNN(nn.Module):
|
||||
"""
|
||||
Bilateral image model using SiameseImageTower.
|
||||
forward(x_od, x_os) -> logits
|
||||
"""
|
||||
|
||||
def __init__(self, *, backbone: str, freeze_ratio: float, num_classes: int, augment: bool):
|
||||
super().__init__()
|
||||
self.tower = SiameseImageTower(
|
||||
backbone=backbone,
|
||||
freeze_ratio=freeze_ratio,
|
||||
augment=augment,
|
||||
use_se=False,
|
||||
)
|
||||
self.head = nn.Linear(self.tower.out_dim, num_classes)
|
||||
|
||||
def forward(self, x_od: torch.Tensor, x_os: torch.Tensor) -> torch.Tensor:
|
||||
return self.head(self.tower(x_od, x_os))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def filter_od_samples(patient_samples: list[dict]) -> list[dict]:
|
||||
"""Extract patient-level OD-only samples (image_1 = OD)."""
|
||||
out = []
|
||||
for s in patient_samples:
|
||||
if s.get("image_1") is not None and s.get("label_1") is not None:
|
||||
out.append({"id_1": s.get("id_1"), "image_1": s["image_1"], "label_1": s["label_1"]})
|
||||
return out
|
||||
|
||||
|
||||
def filter_bilateral_samples(patient_samples: list[dict]) -> list[dict]:
|
||||
return [
|
||||
s for s in patient_samples
|
||||
if s.get("image_1") is not None
|
||||
and s.get("image_2") is not None
|
||||
and s.get("label_1") is not None
|
||||
]
|
||||
|
||||
|
||||
def make_loader(samples, slots, *, image_transform, batch_size, shuffle, num_workers) -> DataLoader:
|
||||
ds = SlotDataset(samples, slots, image_transform=image_transform)
|
||||
return DataLoader(
|
||||
ds,
|
||||
batch_size=batch_size,
|
||||
shuffle=shuffle,
|
||||
num_workers=num_workers,
|
||||
collate_fn=slot_collate,
|
||||
)
|
||||
|
||||
|
||||
def to_label_tensor(labels, device: torch.device) -> torch.Tensor:
|
||||
if torch.is_tensor(labels):
|
||||
return labels.to(device=device, dtype=torch.long)
|
||||
return torch.as_tensor(labels, dtype=torch.long, device=device)
|
||||
|
||||
|
||||
def _drop_mixed_label_patients(df, *, patient_col: str, label_col: str):
|
||||
import pandas as pd
|
||||
per_patient = (
|
||||
df.groupby(patient_col)[label_col]
|
||||
.agg(lambda s: set(pd.to_numeric(s, errors="coerce").dropna().astype(int).tolist()))
|
||||
)
|
||||
mixed = [pid for pid, labels in per_patient.items() if len(labels) > 1]
|
||||
if not mixed:
|
||||
return df, []
|
||||
return df[~df[patient_col].isin(mixed)].reset_index(drop=True), mixed
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Score helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _score(y_true_chunks, y_prob_chunks, num_classes: int):
|
||||
if not y_true_chunks:
|
||||
return float("nan"), float("nan"), 0
|
||||
y = np.concatenate(y_true_chunks)
|
||||
p = np.concatenate(y_prob_chunks)
|
||||
acc = float((p.argmax(1) == y).mean())
|
||||
try:
|
||||
auc = (
|
||||
float(roc_auc_score(y, p[:, 1]))
|
||||
if num_classes == 2
|
||||
else float(roc_auc_score(y, p, multi_class="ovr", average="macro"))
|
||||
)
|
||||
except Exception:
|
||||
auc = float("nan")
|
||||
return acc, auc, int(len(y))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Train / evaluate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def train_baseline_epoch(model, loader, opt, device):
|
||||
model.train()
|
||||
total_loss = total_correct = total_n = 0
|
||||
for batch in loader:
|
||||
x = batch.get("image_1")
|
||||
y = batch.get("label_1")
|
||||
if not torch.is_tensor(x):
|
||||
continue
|
||||
y = to_label_tensor(y, device)
|
||||
x = x.to(device)
|
||||
logits = model(x)
|
||||
loss = F.cross_entropy(logits, y)
|
||||
opt.zero_grad()
|
||||
loss.backward()
|
||||
opt.step()
|
||||
bs = y.shape[0]
|
||||
total_loss += float(loss.item()) * bs
|
||||
total_correct += int((logits.argmax(1) == y).sum())
|
||||
total_n += bs
|
||||
return (total_loss / total_n if total_n else float("nan"),
|
||||
total_correct / total_n if total_n else float("nan"))
|
||||
|
||||
|
||||
def train_siamese_epoch(model, loader, opt, device):
|
||||
model.train()
|
||||
total_loss = total_correct = total_n = 0
|
||||
for batch in loader:
|
||||
x1 = batch.get("image_1")
|
||||
x2 = batch.get("image_2")
|
||||
y = batch.get("label_1")
|
||||
if not torch.is_tensor(x1) or not torch.is_tensor(x2):
|
||||
continue
|
||||
y = to_label_tensor(y, device)
|
||||
logits = model(x1.to(device), x2.to(device))
|
||||
loss = F.cross_entropy(logits, y)
|
||||
opt.zero_grad()
|
||||
loss.backward()
|
||||
opt.step()
|
||||
bs = y.shape[0]
|
||||
total_loss += float(loss.item()) * bs
|
||||
total_correct += int((logits.argmax(1) == y).sum())
|
||||
total_n += bs
|
||||
return (total_loss / total_n if total_n else float("nan"),
|
||||
total_correct / total_n if total_n else float("nan"))
|
||||
|
||||
|
||||
def evaluate_baseline(model, loader, device, num_classes):
|
||||
model.eval()
|
||||
y_true, y_prob = [], []
|
||||
with torch.no_grad():
|
||||
for batch in loader:
|
||||
x = batch.get("image_1")
|
||||
y = batch.get("label_1")
|
||||
if not torch.is_tensor(x):
|
||||
continue
|
||||
y_t = to_label_tensor(y, device)
|
||||
p = F.softmax(model(x.to(device)), dim=1).cpu().numpy()
|
||||
y_true.append(y_t.cpu().numpy())
|
||||
y_prob.append(p)
|
||||
return _score(y_true, y_prob, num_classes)
|
||||
|
||||
|
||||
def evaluate_siamese(model, loader, device, num_classes):
|
||||
model.eval()
|
||||
y_true, y_prob = [], []
|
||||
with torch.no_grad():
|
||||
for batch in loader:
|
||||
x1 = batch.get("image_1")
|
||||
x2 = batch.get("image_2")
|
||||
y = batch.get("label_1")
|
||||
if not torch.is_tensor(x1) or not torch.is_tensor(x2):
|
||||
continue
|
||||
y_t = to_label_tensor(y, device)
|
||||
p = F.softmax(model(x1.to(device), x2.to(device)), dim=1).cpu().numpy()
|
||||
y_true.append(y_t.cpu().numpy())
|
||||
y_prob.append(p)
|
||||
return _score(y_true, y_prob, num_classes)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Result dataclass
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class FoldResult:
|
||||
mode: str
|
||||
fold: int
|
||||
# Best-epoch validation metrics
|
||||
best_epoch: int
|
||||
baseline_best_val_auc: float
|
||||
baseline_best_val_acc: float
|
||||
siamese_best_val_auc: float
|
||||
siamese_best_val_acc: float
|
||||
# Holdout metrics at the respective best-epoch checkpoint
|
||||
baseline_holdout_auc: float
|
||||
baseline_holdout_acc: float
|
||||
baseline_holdout_n: int
|
||||
siamese_holdout_auc: float
|
||||
siamese_holdout_acc: float
|
||||
siamese_holdout_n: int
|
||||
# Sample sizes
|
||||
baseline_n: int
|
||||
siamese_n: int
|
||||
|
||||
|
||||
def _nan() -> float:
|
||||
return float("nan")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main fold runner
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def run_fold(
|
||||
fold: int,
|
||||
split,
|
||||
mode: str,
|
||||
args,
|
||||
device: torch.device,
|
||||
data,
|
||||
num_classes: int,
|
||||
profile_od,
|
||||
profile_patient,
|
||||
fold_dir: Path,
|
||||
) -> FoldResult:
|
||||
holdout_df = split.holdout
|
||||
|
||||
# ---- build samples ----
|
||||
bilat_train = filter_bilateral_samples(profile_patient.build_samples(df=split.train, clinical=data))
|
||||
bilat_val = filter_bilateral_samples(profile_patient.build_samples(df=split.val, clinical=data))
|
||||
od_train = filter_od_samples(bilat_train)
|
||||
od_val = filter_od_samples(bilat_val)
|
||||
|
||||
bilat_holdout = []
|
||||
od_holdout = []
|
||||
if holdout_df is not None and not holdout_df.empty:
|
||||
bilat_holdout = filter_bilateral_samples(profile_patient.build_samples(df=holdout_df, clinical=data))
|
||||
od_holdout = filter_od_samples(bilat_holdout)
|
||||
|
||||
# ---- models ----
|
||||
baseline = BaselineCNN(
|
||||
backbone=args.backbone, freeze_ratio=args.freeze_ratio,
|
||||
num_classes=num_classes, augment=args.augment,
|
||||
).to(device)
|
||||
siamese = SiameseCNN(
|
||||
backbone=args.backbone, freeze_ratio=args.freeze_ratio,
|
||||
num_classes=num_classes, augment=args.augment,
|
||||
).to(device)
|
||||
|
||||
slots_od = profile_od.slot_descriptors()
|
||||
slots_patient = profile_patient.slot_descriptors()
|
||||
|
||||
# ---- loaders ----
|
||||
loader_kw = dict(batch_size=args.batch_size, num_workers=args.num_workers)
|
||||
train_base = make_loader(od_train, slots_od, image_transform=baseline.tower.transform, shuffle=True, **loader_kw)
|
||||
val_base = make_loader(od_val, slots_od, image_transform=baseline.tower.transform, shuffle=False, **loader_kw)
|
||||
train_siam = make_loader(bilat_train, slots_patient, image_transform=siamese.tower.transform, shuffle=True, **loader_kw)
|
||||
val_siam = make_loader(bilat_val, slots_patient, image_transform=siamese.tower.transform, shuffle=False, **loader_kw)
|
||||
|
||||
ho_base = (
|
||||
make_loader(od_holdout, slots_od, image_transform=baseline.tower.transform, shuffle=False, **loader_kw)
|
||||
if od_holdout else None
|
||||
)
|
||||
ho_siam = (
|
||||
make_loader(bilat_holdout, slots_patient, image_transform=siamese.tower.transform, shuffle=False, **loader_kw)
|
||||
if bilat_holdout else None
|
||||
)
|
||||
|
||||
opt_base = torch.optim.Adam(baseline.parameters(), lr=args.lr)
|
||||
opt_siam = torch.optim.Adam(siamese.parameters(), lr=args.lr)
|
||||
|
||||
# ---- epoch log ----
|
||||
epoch_log_path = fold_dir / "epoch_log.csv"
|
||||
epoch_fields = [
|
||||
"fold", "epoch",
|
||||
"base_train_loss", "base_train_acc",
|
||||
"base_val_auc", "base_val_acc", "base_val_n",
|
||||
"siam_train_loss", "siam_train_acc",
|
||||
"siam_val_auc", "siam_val_acc", "siam_val_n",
|
||||
"ho_base_auc", "ho_base_acc", "ho_base_n",
|
||||
"ho_siam_auc", "ho_siam_acc", "ho_siam_n",
|
||||
]
|
||||
epoch_fp = epoch_log_path.open("w", newline="", encoding="utf-8")
|
||||
epoch_writer = csv.DictWriter(epoch_fp, fieldnames=epoch_fields)
|
||||
epoch_writer.writeheader()
|
||||
|
||||
def _f(v):
|
||||
return None if (v is None or (isinstance(v, float) and np.isnan(v))) else round(float(v), 6)
|
||||
|
||||
# ---- best-epoch tracking ----
|
||||
best_base_auc = -1.0
|
||||
best_siam_auc = -1.0
|
||||
best_base_state: Optional[dict] = None
|
||||
best_siam_state: Optional[dict] = None
|
||||
best_base_val_acc = _nan()
|
||||
best_siam_val_acc = _nan()
|
||||
# Holdout metrics snapped at best-val checkpoint
|
||||
snap_ho_base_auc = _nan()
|
||||
snap_ho_base_acc = _nan()
|
||||
snap_ho_base_n = 0
|
||||
snap_ho_siam_auc = _nan()
|
||||
snap_ho_siam_acc = _nan()
|
||||
snap_ho_siam_n = 0
|
||||
best_epoch = 0
|
||||
|
||||
print(
|
||||
f" [fold {fold+1}] training {args.epochs} epochs | "
|
||||
f"baseline n_train={len(od_train)} n_val={len(od_val)} | "
|
||||
f"siamese n_train={len(bilat_train)} n_val={len(bilat_val)}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
for epoch in range(args.epochs):
|
||||
bl_loss, bl_acc = train_baseline_epoch(baseline, train_base, opt_base, device)
|
||||
si_loss, si_acc = train_siamese_epoch(siamese, train_siam, opt_siam, device)
|
||||
|
||||
b_val_acc, b_val_auc, b_val_n = evaluate_baseline(baseline, val_base, device, num_classes)
|
||||
s_val_acc, s_val_auc, s_val_n = evaluate_siamese( siamese, val_siam, device, num_classes)
|
||||
|
||||
# Holdout at this epoch (always evaluated for logging, cheaply)
|
||||
hb_auc, hb_acc, hb_n = (_nan(), _nan(), 0)
|
||||
hs_auc, hs_acc, hs_n = (_nan(), _nan(), 0)
|
||||
if ho_base is not None:
|
||||
hb_acc, hb_auc, hb_n = evaluate_baseline(baseline, ho_base, device, num_classes)
|
||||
if ho_siam is not None:
|
||||
hs_acc, hs_auc, hs_n = evaluate_siamese(siamese, ho_siam, device, num_classes)
|
||||
|
||||
# Best-epoch tracking: snapshot state independently per model
|
||||
if not np.isnan(b_val_auc) and b_val_auc > best_base_auc:
|
||||
best_base_auc = b_val_auc
|
||||
best_base_val_acc = b_val_acc
|
||||
best_base_state = copy.deepcopy(baseline.state_dict())
|
||||
snap_ho_base_auc = hb_auc
|
||||
snap_ho_base_acc = hb_acc
|
||||
snap_ho_base_n = hb_n
|
||||
|
||||
if not np.isnan(s_val_auc) and s_val_auc > best_siam_auc:
|
||||
best_siam_auc = s_val_auc
|
||||
best_siam_val_acc = s_val_acc
|
||||
best_siam_state = copy.deepcopy(siamese.state_dict())
|
||||
snap_ho_siam_auc = hs_auc
|
||||
snap_ho_siam_acc = hs_acc
|
||||
snap_ho_siam_n = hs_n
|
||||
best_epoch = epoch + 1
|
||||
|
||||
row = {
|
||||
"fold": fold, "epoch": epoch + 1,
|
||||
"base_train_loss": _f(bl_loss), "base_train_acc": _f(bl_acc),
|
||||
"base_val_auc": _f(b_val_auc), "base_val_acc": _f(b_val_acc), "base_val_n": b_val_n,
|
||||
"siam_train_loss": _f(si_loss), "siam_train_acc": _f(si_acc),
|
||||
"siam_val_auc": _f(s_val_auc), "siam_val_acc": _f(s_val_acc), "siam_val_n": s_val_n,
|
||||
"ho_base_auc": _f(hb_auc), "ho_base_acc": _f(hb_acc), "ho_base_n": hb_n,
|
||||
"ho_siam_auc": _f(hs_auc), "ho_siam_acc": _f(hs_acc), "ho_siam_n": hs_n,
|
||||
}
|
||||
epoch_writer.writerow(row)
|
||||
epoch_fp.flush()
|
||||
|
||||
if args.log_every > 0 and (epoch + 1) % args.log_every == 0:
|
||||
print(
|
||||
f" ep {epoch+1:>3}/{args.epochs} "
|
||||
f"base val AUC={b_val_auc:.4f} siam val AUC={s_val_auc:.4f} "
|
||||
f"(best base={best_base_auc:.4f} best siam={best_siam_auc:.4f})",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
epoch_fp.close()
|
||||
|
||||
# Save best checkpoints
|
||||
if best_base_state is not None:
|
||||
torch.save(best_base_state, fold_dir / "best_baseline.pt")
|
||||
if best_siam_state is not None:
|
||||
torch.save(best_siam_state, fold_dir / "best_siamese.pt")
|
||||
|
||||
result = FoldResult(
|
||||
mode=mode, fold=fold,
|
||||
best_epoch=best_epoch,
|
||||
baseline_best_val_auc=best_base_auc,
|
||||
baseline_best_val_acc=best_base_val_acc,
|
||||
siamese_best_val_auc=best_siam_auc,
|
||||
siamese_best_val_acc=best_siam_val_acc,
|
||||
baseline_holdout_auc=snap_ho_base_auc,
|
||||
baseline_holdout_acc=snap_ho_base_acc,
|
||||
baseline_holdout_n=snap_ho_base_n,
|
||||
siamese_holdout_auc=snap_ho_siam_auc,
|
||||
siamese_holdout_acc=snap_ho_siam_acc,
|
||||
siamese_holdout_n=snap_ho_siam_n,
|
||||
baseline_n=len(od_val),
|
||||
siamese_n=len(bilat_val),
|
||||
)
|
||||
|
||||
print(
|
||||
f" [fold {fold+1}] BEST "
|
||||
f"base val AUC={best_base_auc:.4f} acc={best_base_val_acc:.4f} "
|
||||
f"siam val AUC={best_siam_auc:.4f} acc={best_siam_val_acc:.4f} "
|
||||
f"(siam best epoch={best_epoch})",
|
||||
flush=True,
|
||||
)
|
||||
if snap_ho_base_n > 0 or snap_ho_siam_n > 0:
|
||||
print(
|
||||
f" [fold {fold+1}] HOUT "
|
||||
f"base AUC={snap_ho_base_auc:.4f} acc={snap_ho_base_acc:.4f} (n={snap_ho_base_n}) "
|
||||
f"siam AUC={snap_ho_siam_auc:.4f} acc={snap_ho_siam_acc:.4f} (n={snap_ho_siam_n})",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _summary(results: list[FoldResult]) -> dict:
|
||||
def _means(vals):
|
||||
v = np.array([x for x in vals if not np.isnan(x)], dtype=float)
|
||||
return (float(np.mean(v)) if len(v) else None,
|
||||
float(np.std(v)) if len(v) else None)
|
||||
|
||||
b_val_aucs = [r.baseline_best_val_auc for r in results]
|
||||
s_val_aucs = [r.siamese_best_val_auc for r in results]
|
||||
b_ho_aucs = [r.baseline_holdout_auc for r in results]
|
||||
s_ho_aucs = [r.siamese_holdout_auc for r in results]
|
||||
b_val_accs = [r.baseline_best_val_acc for r in results]
|
||||
s_val_accs = [r.siamese_best_val_acc for r in results]
|
||||
b_ho_accs = [r.baseline_holdout_acc for r in results]
|
||||
s_ho_accs = [r.siamese_holdout_acc for r in results]
|
||||
|
||||
deltas_val_auc = [s - b for b, s in zip(b_val_aucs, s_val_aucs)
|
||||
if not np.isnan(b) and not np.isnan(s)]
|
||||
deltas_ho_auc = [s - b for b, s in zip(b_ho_aucs, s_ho_aucs)
|
||||
if not np.isnan(b) and not np.isnan(s)]
|
||||
|
||||
bva_m, bva_s = _means(b_val_aucs)
|
||||
sva_m, sva_s = _means(s_val_aucs)
|
||||
bha_m, bha_s = _means(b_ho_aucs)
|
||||
sha_m, sha_s = _means(s_ho_aucs)
|
||||
|
||||
return {
|
||||
"baseline_best_val": {"auc_mean": bva_m, "auc_std": bva_s, "acc_mean": _means(b_val_accs)[0]},
|
||||
"siamese_best_val": {"auc_mean": sva_m, "auc_std": sva_s, "acc_mean": _means(s_val_accs)[0]},
|
||||
"delta_val_auc": {"mean": float(np.mean(deltas_val_auc)) if deltas_val_auc else None,
|
||||
"std": float(np.std(deltas_val_auc)) if deltas_val_auc else None},
|
||||
"baseline_holdout": {"auc_mean": bha_m, "auc_std": bha_s, "acc_mean": _means(b_ho_accs)[0]},
|
||||
"siamese_holdout": {"auc_mean": sha_m, "auc_std": sha_s, "acc_mean": _means(s_ho_accs)[0]},
|
||||
"delta_holdout_auc": {"mean": float(np.mean(deltas_ho_auc)) if deltas_ho_auc else None,
|
||||
"std": float(np.std(deltas_ho_auc)) if deltas_ho_auc else None},
|
||||
}
|
||||
|
||||
|
||||
def _print_summary(mode: str, s: dict) -> None:
|
||||
def f(v):
|
||||
return "nan" if v is None else f"{v:.4f}"
|
||||
|
||||
bv = s["baseline_best_val"]
|
||||
sv = s["siamese_best_val"]
|
||||
dv = s["delta_val_auc"]
|
||||
bh = s["baseline_holdout"]
|
||||
sh = s["siamese_holdout"]
|
||||
dh = s["delta_holdout_auc"]
|
||||
|
||||
print(f"\n=== Summary [{mode}] (best-epoch metrics) ===")
|
||||
print(f" val baseline AUC={f(bv['auc_mean'])}±{f(bv['auc_std'])} acc={f(bv['acc_mean'])}")
|
||||
print(f" val siamese AUC={f(sv['auc_mean'])}±{f(sv['auc_std'])} acc={f(sv['acc_mean'])}")
|
||||
print(f" val delta AUC={f(dv['mean'])}±{f(dv['std'])}")
|
||||
print(f" hout baseline AUC={f(bh['auc_mean'])}±{f(bh['auc_std'])} acc={f(bh['acc_mean'])}")
|
||||
print(f" hout siamese AUC={f(sh['auc_mean'])}±{f(sh['auc_std'])} acc={f(sh['acc_mean'])}")
|
||||
print(f" hout delta AUC={f(dh['mean'])}±{f(dh['std'])}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def parse_args():
|
||||
ap = argparse.ArgumentParser(
|
||||
description="Baseline single-eye vs SiameseImageTower bilateral comparison."
|
||||
)
|
||||
ap.add_argument("--image-dir", default="Papila/FundusImages")
|
||||
ap.add_argument("--clinical-dir", default="Papila/ClinicalData")
|
||||
ap.add_argument("--label-col", default="Diagnosis")
|
||||
ap.add_argument("--cat-cols", nargs="*", default=["Gender", "Phakic/Pseudophakic"])
|
||||
ap.add_argument("--eval-mode", choices=["binary", "multiclass"], default="binary")
|
||||
ap.add_argument(
|
||||
"--eval-modes", nargs="+", choices=["binary", "multiclass"], default=None,
|
||||
help="Run multiple modes in one pass, e.g. --eval-modes binary multiclass",
|
||||
)
|
||||
ap.add_argument("--n-splits", type=int, default=5)
|
||||
ap.add_argument("--fold-seed", type=int, default=42)
|
||||
ap.add_argument("--holdout-per-class", type=int, default=0)
|
||||
ap.add_argument("--holdout-seed", type=int, default=123)
|
||||
ap.add_argument("--folds", type=int, default=5)
|
||||
ap.add_argument("--epochs", type=int, default=40)
|
||||
ap.add_argument("--batch-size", type=int, default=8)
|
||||
ap.add_argument("--lr", type=float, default=1e-4)
|
||||
ap.add_argument("--backbone", default="refugelike")
|
||||
ap.add_argument("--freeze-ratio", type=float, default=0.0)
|
||||
ap.add_argument("--augment", action="store_true")
|
||||
ap.add_argument("--num-workers", type=int, default=0)
|
||||
ap.add_argument("--device", choices=["auto", "cpu", "cuda"], default="auto")
|
||||
ap.add_argument("--seed", type=int, default=1234)
|
||||
ap.add_argument("--run-name", default=None)
|
||||
ap.add_argument("--output-root", default="analysis_data/basic_analysis")
|
||||
ap.add_argument(
|
||||
"--exclude-mixed-patients", action="store_true",
|
||||
help="Drop patients whose two eyes have different labels before splitting.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--log-every", type=int, default=5,
|
||||
help="Print epoch progress every N epochs (0 to disable).",
|
||||
)
|
||||
return ap.parse_args()
|
||||
|
||||
|
||||
def choose_device(name: str) -> torch.device:
|
||||
if name == "cuda":
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("--device cuda requested but CUDA is not available.")
|
||||
return torch.device("cuda")
|
||||
if name == "cpu":
|
||||
return torch.device("cpu")
|
||||
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
device = choose_device(args.device)
|
||||
seed_everything(args.seed)
|
||||
|
||||
print(f"Device: {device}", flush=True)
|
||||
print("Loading PAPILA data...", flush=True)
|
||||
data = build_papila_data(
|
||||
image_dir=args.image_dir,
|
||||
clinical_dir=args.clinical_dir,
|
||||
label_col=args.label_col,
|
||||
cat_cols=list(args.cat_cols),
|
||||
n_splits=args.n_splits,
|
||||
random_seed=args.fold_seed,
|
||||
)
|
||||
print(f"Loaded: {len(data.df)} rows", flush=True)
|
||||
|
||||
ts = time.strftime("%Y%m%d_%H%M%S")
|
||||
run_name = args.run_name or f"siamese_compare_{ts}"
|
||||
out_dir = Path(args.output_root) / run_name
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
eval_modes = args.eval_modes if args.eval_modes else [args.eval_mode]
|
||||
|
||||
all_results: dict[str, list[FoldResult]] = {}
|
||||
summaries: dict[str, dict] = {}
|
||||
|
||||
for mode in eval_modes:
|
||||
import pandas as pd
|
||||
df_mode = data.df.copy()
|
||||
|
||||
if args.exclude_mixed_patients:
|
||||
before = df_mode["Patient ID"].nunique()
|
||||
df_mode, mixed = _drop_mixed_label_patients(
|
||||
df_mode, patient_col="Patient ID", label_col=args.label_col
|
||||
)
|
||||
print(f"[{mode}] dropped {len(mixed)} mixed-label patients "
|
||||
f"({before} -> {df_mode['Patient ID'].nunique()})", flush=True)
|
||||
|
||||
if mode == "binary":
|
||||
df_mode = df_mode[df_mode[args.label_col].isin([0, 1])].reset_index(drop=True)
|
||||
|
||||
num_classes = 2 if mode == "binary" else int(df_mode[args.label_col].nunique())
|
||||
print(
|
||||
f"\n[{mode}] num_classes={num_classes} rows={len(df_mode)} "
|
||||
f"patients={df_mode['Patient ID'].nunique()}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
split_manager = PatientFirstSplitManager(
|
||||
patient_col="Patient ID", label_col=args.label_col
|
||||
)
|
||||
split_args = SimpleNamespace(
|
||||
eval_mode=mode,
|
||||
holdout_per_class=args.holdout_per_class,
|
||||
holdout_seed=args.holdout_seed,
|
||||
n_splits=args.n_splits,
|
||||
fold_seed=args.fold_seed,
|
||||
)
|
||||
clinical_ns = SimpleNamespace(df=df_mode, label_col=args.label_col)
|
||||
plans = split_manager.build_plans(clinical=clinical_ns, args=split_args, profile=None)
|
||||
n_folds = min(args.folds, len(plans))
|
||||
|
||||
profile_od = build_papila_profile(
|
||||
patient_col="Patient ID", label_col=args.label_col, sample_mode="eye"
|
||||
)
|
||||
profile_patient = build_papila_profile(
|
||||
patient_col="Patient ID", label_col=args.label_col, sample_mode="patient"
|
||||
)
|
||||
|
||||
mode_dir = out_dir / mode
|
||||
mode_dir.mkdir(exist_ok=True)
|
||||
|
||||
fold_results: list[FoldResult] = []
|
||||
for fold in range(n_folds):
|
||||
fold_seed = args.seed + fold * 100
|
||||
seed_everything(fold_seed)
|
||||
fold_dir = mode_dir / f"fold{fold}"
|
||||
fold_dir.mkdir(exist_ok=True)
|
||||
|
||||
print(f"\n[{mode}] fold {fold+1}/{n_folds}", flush=True)
|
||||
result = run_fold(
|
||||
fold=fold,
|
||||
split=plans[fold],
|
||||
mode=mode,
|
||||
args=args,
|
||||
device=device,
|
||||
data=data,
|
||||
num_classes=num_classes,
|
||||
profile_od=profile_od,
|
||||
profile_patient=profile_patient,
|
||||
fold_dir=fold_dir,
|
||||
)
|
||||
fold_results.append(result)
|
||||
|
||||
# Write per-mode CSV
|
||||
fold_csv = out_dir / f"{mode}_fold_results.csv"
|
||||
csv_fields = list(FoldResult.__dataclass_fields__.keys())
|
||||
with fold_csv.open("w", newline="", encoding="utf-8") as fh:
|
||||
w = csv.DictWriter(fh, fieldnames=csv_fields)
|
||||
w.writeheader()
|
||||
for r in fold_results:
|
||||
w.writerow({k: getattr(r, k) for k in csv_fields})
|
||||
|
||||
summary = _summary(fold_results)
|
||||
_print_summary(mode, summary)
|
||||
|
||||
all_results[mode] = fold_results
|
||||
summaries[mode] = summary
|
||||
|
||||
payload = {
|
||||
"run_name": run_name,
|
||||
"timestamp": ts,
|
||||
"config": vars(args),
|
||||
"summaries": summaries,
|
||||
}
|
||||
(out_dir / "summary.json").write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
||||
print(f"\nOutputs written to: {out_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
+264
@@ -0,0 +1,264 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Univariate ROC curves for PAPILA clinical variables."""
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Iterable, List, Tuple
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.metrics import roc_curve, auc
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from classes import build_papila_clinical
|
||||
|
||||
|
||||
# ---------------------------
|
||||
# Config (edit in IDE)
|
||||
# ---------------------------
|
||||
IMAGE_DIR = "Papila/FundusImages"
|
||||
CLINICAL_DIR = "Papila/ClinicalData"
|
||||
LABEL_COL = "Diagnosis"
|
||||
CAT_COLS = ["Gender", "Phakic/Pseudophakic"]
|
||||
EXCLUDE_COLS = {"Pneumatic", "Perkins"}
|
||||
INCLUDE_CATEGORICAL = False
|
||||
POSITIVE_LABEL = 1
|
||||
NEGATIVE_LABEL = 0
|
||||
DROP_LABELS = [2]
|
||||
OUTPUT_DIR = Path("analysis_data/basic_analysis/papila_univariate_roc")
|
||||
DEBUG_PRINTS = False
|
||||
PLOT_PER_FEATURE = False
|
||||
DI_OPTRE_COL_PREFIXES = ("dioptre",)
|
||||
ADD_DIOPTRE_ABS = True
|
||||
ADD_DIOPTRE_SQUARED = True
|
||||
|
||||
|
||||
def _sanitize(name: str) -> str:
|
||||
safe = re.sub(r"[^A-Za-z0-9._-]+", "_", str(name)).strip("_")
|
||||
return safe or "var"
|
||||
|
||||
|
||||
def _select_binary_labels(labels: pd.Series,
|
||||
positive_label: int,
|
||||
negative_label: int,
|
||||
drop_labels: Iterable[int]) -> Tuple[np.ndarray, np.ndarray]:
|
||||
labels_num = pd.to_numeric(labels, errors="coerce")
|
||||
use_num = labels_num.notna().any()
|
||||
lab = labels_num if use_num else labels.astype(str)
|
||||
|
||||
drop_set = set(drop_labels or [])
|
||||
keep = lab.isin([positive_label, negative_label])
|
||||
if drop_set:
|
||||
keep &= ~lab.isin(drop_set)
|
||||
|
||||
y = (lab == positive_label).astype(int)
|
||||
return y.values, keep.values
|
||||
|
||||
|
||||
def _compute_roc(y: np.ndarray, scores: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray, float]:
|
||||
fpr, tpr, thresholds = roc_curve(y, scores, pos_label=1)
|
||||
auc_val = float(auc(fpr, tpr))
|
||||
return fpr, tpr, thresholds, auc_val
|
||||
|
||||
|
||||
def _best_threshold(fpr: np.ndarray, tpr: np.ndarray, thresholds: np.ndarray) -> Tuple[float, float, float]:
|
||||
youden = tpr - fpr
|
||||
idx = int(np.nanargmax(youden))
|
||||
return float(thresholds[idx]), float(tpr[idx]), float(fpr[idx])
|
||||
|
||||
|
||||
def _plot_roc(fpr: np.ndarray, tpr: np.ndarray, auc_val: float, title: str, out_path: Path) -> None:
|
||||
fig, ax = plt.subplots(figsize=(5.5, 4.5))
|
||||
ax.plot(fpr, tpr, lw=1.8, label=f"AUC={auc_val:.3f}")
|
||||
ax.plot([0, 1], [0, 1], "k--", lw=1)
|
||||
ax.set_xlabel("False Positive Rate")
|
||||
ax.set_ylabel("True Positive Rate")
|
||||
ax.set_title(title)
|
||||
ax.legend(loc="lower right")
|
||||
ax.grid(True, alpha=0.3, linestyle="--")
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=170)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def _plot_overlay(curves, title: str, out_path: Path) -> None:
|
||||
fig, ax = plt.subplots(figsize=(7, 5.5))
|
||||
cmap = plt.get_cmap("tab20")
|
||||
for i, (name, fpr, tpr, auc_val) in enumerate(curves):
|
||||
color = cmap(i % cmap.N)
|
||||
ax.plot(fpr, tpr, lw=1.6, color=color, label=f"{name} (AUC={auc_val:.3f})")
|
||||
ax.plot([0, 1], [0, 1], "k--", lw=1)
|
||||
ax.set_xlabel("False Positive Rate")
|
||||
ax.set_ylabel("True Positive Rate")
|
||||
ax.set_title(title)
|
||||
ax.legend(loc="upper left", fontsize="small")
|
||||
ax.grid(True, alpha=0.3, linestyle="--")
|
||||
fig.tight_layout()
|
||||
fig.savefig(out_path, dpi=170)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def _iter_numeric(df: pd.DataFrame, cols: List[str]):
|
||||
for col in cols:
|
||||
if col not in df.columns:
|
||||
continue
|
||||
s = pd.to_numeric(df[col], errors="coerce")
|
||||
yield col, s
|
||||
|
||||
|
||||
def _is_dioptre_col(col: str) -> bool:
|
||||
name = str(col).strip().lower()
|
||||
return any(name.startswith(prefix) for prefix in DI_OPTRE_COL_PREFIXES)
|
||||
|
||||
|
||||
def _iter_numeric_with_transforms(df: pd.DataFrame, cols: List[str]):
|
||||
for col, s in _iter_numeric(df, cols):
|
||||
yield col, s
|
||||
if _is_dioptre_col(col):
|
||||
if ADD_DIOPTRE_ABS:
|
||||
yield f"{col}_abs", s.abs()
|
||||
if ADD_DIOPTRE_SQUARED:
|
||||
yield f"{col}_sq", s.pow(2)
|
||||
|
||||
|
||||
def _include_in_overlay(feature_name: str) -> bool:
|
||||
name = str(feature_name).strip().lower()
|
||||
if _is_dioptre_col(name) and not name.endswith("_abs"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _iter_categorical(df: pd.DataFrame, cols: List[str]):
|
||||
for col in cols:
|
||||
if col not in df.columns:
|
||||
continue
|
||||
s = df[col]
|
||||
vals = s.dropna().unique().tolist()
|
||||
try:
|
||||
vals = sorted(vals)
|
||||
except Exception:
|
||||
pass
|
||||
for v in vals:
|
||||
name = f"{col}=={v}"
|
||||
ind = (s == v).astype(int)
|
||||
yield name, ind
|
||||
|
||||
|
||||
def main() -> None:
|
||||
clinical = build_papila_clinical(
|
||||
image_dir=IMAGE_DIR,
|
||||
clinical_dir=CLINICAL_DIR,
|
||||
label_col=LABEL_COL,
|
||||
cat_cols=CAT_COLS,
|
||||
)
|
||||
|
||||
df = clinical.df.copy()
|
||||
labels = df[LABEL_COL]
|
||||
y_all, keep_mask = _select_binary_labels(labels, POSITIVE_LABEL, NEGATIVE_LABEL, DROP_LABELS)
|
||||
|
||||
out_dir = OUTPUT_DIR
|
||||
plot_dir = out_dir / "plots"
|
||||
plot_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
rows = []
|
||||
curves = []
|
||||
|
||||
base_exclude = {LABEL_COL, "Patient ID"} | EXCLUDE_COLS | set(CAT_COLS)
|
||||
if "eyeID" not in CAT_COLS:
|
||||
base_exclude.add("eyeID")
|
||||
candidate_cols = [c for c in df.columns if c not in base_exclude]
|
||||
numeric_cols = []
|
||||
for col in candidate_cols:
|
||||
s = pd.to_numeric(df[col], errors="coerce")
|
||||
if s.notna().any():
|
||||
numeric_cols.append(col)
|
||||
|
||||
if DEBUG_PRINTS:
|
||||
for col in ("IOP_raw", "IOP_corr"):
|
||||
if col not in df.columns:
|
||||
print(f"[debug] {col} missing from df")
|
||||
continue
|
||||
s = pd.to_numeric(df[col], errors="coerce")
|
||||
print(f"[debug] {col}: non-null={int(s.notna().sum())}, unique={int(s.nunique(dropna=True))}")
|
||||
for col, series in _iter_numeric_with_transforms(df, numeric_cols):
|
||||
mask = keep_mask & series.notna().values
|
||||
y = y_all[mask]
|
||||
scores = series.values[mask].astype(float)
|
||||
if y.size < 2 or np.unique(y).size < 2:
|
||||
continue
|
||||
if np.nanmin(scores) == np.nanmax(scores):
|
||||
continue
|
||||
fpr, tpr, thresholds, auc_val = _compute_roc(y, scores)
|
||||
thr, best_tpr, best_fpr = _best_threshold(fpr, tpr, thresholds)
|
||||
direction = "high" if auc_val >= 0.5 else "low"
|
||||
title = f"{col} (n={y.size}, direction={direction})"
|
||||
if PLOT_PER_FEATURE:
|
||||
out_path = plot_dir / f"roc_{_sanitize(col)}.png"
|
||||
_plot_roc(fpr, tpr, auc_val, title, out_path)
|
||||
if _include_in_overlay(col):
|
||||
curves.append((col, fpr, tpr, auc_val))
|
||||
rows.append({
|
||||
"feature": col,
|
||||
"kind": "numeric",
|
||||
"n": int(y.size),
|
||||
"auc": auc_val,
|
||||
"direction": direction,
|
||||
"best_threshold": thr,
|
||||
"best_tpr": best_tpr,
|
||||
"best_fpr": best_fpr,
|
||||
"best_specificity": 1.0 - best_fpr,
|
||||
})
|
||||
|
||||
if INCLUDE_CATEGORICAL:
|
||||
cat_cols_use = [c for c in clinical.cat_cols if c not in EXCLUDE_COLS]
|
||||
for name, ind in _iter_categorical(df, cat_cols_use):
|
||||
mask = keep_mask & ind.notna().values
|
||||
y = y_all[mask]
|
||||
scores = ind.values[mask].astype(float)
|
||||
if y.size < 2 or np.unique(y).size < 2:
|
||||
continue
|
||||
if np.nanmin(scores) == np.nanmax(scores):
|
||||
continue
|
||||
fpr, tpr, thresholds, auc_val = _compute_roc(y, scores)
|
||||
thr, best_tpr, best_fpr = _best_threshold(fpr, tpr, thresholds)
|
||||
direction = "high" if auc_val >= 0.5 else "low"
|
||||
title = f"{name} (n={y.size}, direction={direction})"
|
||||
if PLOT_PER_FEATURE:
|
||||
out_path = plot_dir / f"roc_{_sanitize(name)}.png"
|
||||
_plot_roc(fpr, tpr, auc_val, title, out_path)
|
||||
if _include_in_overlay(name):
|
||||
curves.append((name, fpr, tpr, auc_val))
|
||||
rows.append({
|
||||
"feature": name,
|
||||
"kind": "categorical",
|
||||
"n": int(y.size),
|
||||
"auc": auc_val,
|
||||
"direction": direction,
|
||||
"best_threshold": thr,
|
||||
"best_tpr": best_tpr,
|
||||
"best_fpr": best_fpr,
|
||||
"best_specificity": 1.0 - best_fpr,
|
||||
})
|
||||
|
||||
if not rows:
|
||||
raise SystemExit("No valid features produced ROC curves. Check labels and feature columns.")
|
||||
|
||||
overlay_path = plot_dir / "roc_overlay.png"
|
||||
_plot_overlay(curves, "Univariate ROC curves", overlay_path)
|
||||
|
||||
out_df = pd.DataFrame(rows).sort_values(by="auc", ascending=False)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_df.to_csv(out_dir / "summary.csv", index=False)
|
||||
print(out_df.to_string(index=False, float_format=lambda x: f"{x:.4f}"))
|
||||
print(f"\nSaved overlay plot to: {overlay_path}")
|
||||
if PLOT_PER_FEATURE:
|
||||
print(f"Saved per-feature plots to: {plot_dir}")
|
||||
print(f"Saved summary to: {out_dir / 'summary.csv'}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user