ryan/hypertower
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

moved_repo_first_update

Browse Source
This commit is contained in:
rpotter6298
2026-02-24 10:39:48 +01:00
commit 9894a23f09
98 changed files with 35387 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/basic_analysis/basic_analysis.py
Executable
+757
View File
@@ -0,0 +1,757 @@
#!/usr/bin/env python3
"""Basic analytics helpers for PAPILA clinical data."""
import re
from pathlib import Path
from typing import Iterable, List, Tuple
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.base import clone
from sklearn.metrics import roc_curve, auc, roc_auc_score, accuracy_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import StratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
from classes import build_papila_clinical
class basic_analytics:
def __init__(
self,
image_dir: str = "Papila/FundusImages",
clinical_dir: str = "Papila/ClinicalData",
label_col: str = "Diagnosis",
cat_cols: Iterable[str] | None = None,
exclude_cols: Iterable[str] | None = None,
positive_label: int = 1,
negative_label: int = 0,
drop_labels: Iterable[int] = (2,),
output_dir: Path | str = Path("analysis_data/basic_analysis"),
debug: bool = False,
) -> None:
self.image_dir = image_dir
self.clinical_dir = clinical_dir
self.label_col = label_col
self.cat_cols = (
list(cat_cols)
if cat_cols is not None
else ["Gender", "Phakic/Pseudophakic"]
)
base_excludes = {"Pneumatic", "Perkins"}
self.exclude_cols = base_excludes | set(exclude_cols or [])
self.positive_label = positive_label
self.negative_label = negative_label
self.drop_labels = list(drop_labels or [])
self.output_dir = Path(output_dir)
self.debug = debug
@staticmethod
def _sanitize(name: str) -> str:
safe = re.sub(r"[^A-Za-z0-9._-]+", "_", str(name)).strip("_")
return safe or "var"
def _build_clinical(self):
return build_papila_clinical(
image_dir=self.image_dir,
clinical_dir=self.clinical_dir,
label_col=self.label_col,
cat_cols=self.cat_cols,
)
def _select_binary_labels(
self,
labels: pd.Series,
) -> Tuple[np.ndarray, np.ndarray]:
labels_num = pd.to_numeric(labels, errors="coerce")
use_num = labels_num.notna().any()
lab = labels_num if use_num else labels.astype(str)
drop_set = set(self.drop_labels or [])
keep = lab.isin([self.positive_label, self.negative_label])
if drop_set:
keep &= ~lab.isin(drop_set)
y = (lab == self.positive_label).astype(int)
return y.values, keep.values
def _base_exclude(self) -> set:
base_exclude = (
{self.label_col, "Patient ID"} | self.exclude_cols | set(self.cat_cols)
)
if "eyeID" not in self.cat_cols:
base_exclude.add("eyeID")
return base_exclude
def _numeric_columns(self, df: pd.DataFrame) -> List[str]:
base_exclude = self._base_exclude()
candidate_cols = [c for c in df.columns if c not in base_exclude]
numeric_cols: List[str] = []
for col in candidate_cols:
s = pd.to_numeric(df[col], errors="coerce")
if s.notna().any():
numeric_cols.append(col)
return numeric_cols
@staticmethod
def _compute_roc(
y: np.ndarray, scores: np.ndarray
) -> Tuple[np.ndarray, np.ndarray, np.ndarray, float]:
fpr, tpr, thresholds = roc_curve(y, scores, pos_label=1)
auc_val = float(auc(fpr, tpr))
return fpr, tpr, thresholds, auc_val
@staticmethod
def _best_threshold(
fpr: np.ndarray, tpr: np.ndarray, thresholds: np.ndarray
) -> Tuple[float, float, float]:
youden = tpr - fpr
idx = int(np.nanargmax(youden))
return float(thresholds[idx]), float(tpr[idx]), float(fpr[idx])
@staticmethod
def _plot_overlay(curves, title: str, out_path: Path) -> None:
fig, ax = plt.subplots(figsize=(7, 5.5))
cmap = plt.get_cmap("tab20")
for i, (name, fpr, tpr, auc_val) in enumerate(curves):
color = cmap(i % cmap.N)
ax.plot(fpr, tpr, lw=1.6, color=color, label=f"{name} (AUC={auc_val:.3f})")
ax.plot([0, 1], [0, 1], "k--", lw=1)
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(title)
ax.legend(loc="upper left", fontsize="small")
ax.grid(True, alpha=0.3, linestyle="--")
fig.tight_layout()
fig.savefig(out_path, dpi=170)
plt.close(fig)
@staticmethod
def _plot_per_feature(
fpr: np.ndarray, tpr: np.ndarray, auc_val: float, title: str, out_path: Path
) -> None:
fig, ax = plt.subplots(figsize=(5.5, 4.5))
ax.plot(fpr, tpr, lw=1.8, label=f"AUC={auc_val:.3f}")
ax.plot([0, 1], [0, 1], "k--", lw=1)
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(title)
ax.legend(loc="lower right")
ax.grid(True, alpha=0.3, linestyle="--")
fig.tight_layout()
fig.savefig(out_path, dpi=170)
plt.close(fig)
@staticmethod
def _plot_roc_line(
fpr: np.ndarray, tpr: np.ndarray, auc_val: float, title: str, out_path: Path
) -> None:
fig, ax = plt.subplots(figsize=(5.5, 4.5))
ax.plot(fpr, tpr, lw=1.8, label=f"AUC={auc_val:.3f}")
ax.plot([0, 1], [0, 1], "k--", lw=1)
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(title)
ax.legend(loc="lower right")
ax.grid(True, alpha=0.3, linestyle="--")
fig.tight_layout()
fig.savefig(out_path, dpi=170)
plt.close(fig)
def _oof_scores(
self,
model,
X: np.ndarray,
y: np.ndarray,
n_splits: int,
random_state: int,
) -> Tuple[np.ndarray, np.ndarray]:
skf = StratifiedKFold(
n_splits=n_splits, shuffle=True, random_state=random_state
)
scores = np.zeros(len(y), dtype=float)
for train_idx, test_idx in skf.split(X, y):
X_train, X_test = X[train_idx], X[test_idx]
y_train = y[train_idx]
if np.unique(y_train).size < 2:
continue
fitted = clone(model)
fitted.fit(X_train, y_train)
if hasattr(fitted, "predict_proba"):
fold_scores = fitted.predict_proba(X_test)[:, 1]
elif hasattr(fitted, "decision_function"):
fold_scores = fitted.decision_function(X_test)
else:
fold_scores = fitted.predict(X_test)
scores[test_idx] = fold_scores
return y.astype(int), scores
def _cv_roc_curves(
self,
model,
X: np.ndarray,
y: np.ndarray,
n_splits: int,
random_state: int,
) -> List[Tuple[np.ndarray, np.ndarray, float]]:
skf = StratifiedKFold(
n_splits=n_splits, shuffle=True, random_state=random_state
)
curves = []
for train_idx, test_idx in skf.split(X, y):
X_train, X_test = X[train_idx], X[test_idx]
y_train, y_test = y[train_idx], y[test_idx]
if np.unique(y_train).size < 2 or np.unique(y_test).size < 2:
continue
fitted = clone(model)
fitted.fit(X_train, y_train)
if hasattr(fitted, "predict_proba"):
scores = fitted.predict_proba(X_test)[:, 1]
elif hasattr(fitted, "decision_function"):
scores = fitted.decision_function(X_test)
else:
scores = fitted.predict(X_test)
fpr, tpr, _ = roc_curve(y_test, scores, pos_label=1)
auc_val = float(auc(fpr, tpr))
curves.append((fpr, tpr, auc_val))
return curves
@staticmethod
def _plot_mean_roc(
curves: List[Tuple[np.ndarray, np.ndarray, float]],
title: str,
out_path: Path,
) -> None:
if not curves:
return
mean_fpr = np.linspace(0.0, 1.0, 200)
tprs = []
aucs = []
for fpr, tpr, auc_val in curves:
tpr_interp = np.interp(mean_fpr, fpr, tpr)
tpr_interp[0] = 0.0
tprs.append(tpr_interp)
aucs.append(auc_val)
mean_tpr = np.mean(tprs, axis=0)
mean_tpr[-1] = 1.0
std_tpr = np.std(tprs, axis=0)
mean_auc = float(np.mean(aucs))
std_auc = float(np.std(aucs, ddof=0))
fig, ax = plt.subplots(figsize=(5.8, 4.6))
ax.plot(mean_fpr, mean_tpr, lw=2, label=f"AUC={mean_auc:.3f}±{std_auc:.3f}")
ax.fill_between(
mean_fpr,
np.maximum(mean_tpr - std_tpr, 0),
np.minimum(mean_tpr + std_tpr, 1),
color="grey",
alpha=0.2,
)
ax.plot([0, 1], [0, 1], "k--", lw=1)
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(title)
ax.legend(loc="lower right")
ax.grid(True, alpha=0.3, linestyle="--")
fig.tight_layout()
fig.savefig(out_path, dpi=170)
plt.close(fig)
def _iter_categorical(self, df: pd.DataFrame, cols: List[str]):
for col in cols:
if col not in df.columns:
continue
s = df[col]
vals = s.dropna().unique().tolist()
try:
vals = sorted(vals)
except Exception:
pass
for v in vals:
name = f"{col}=={v}"
ind = (s == v).astype(int)
yield name, ind
def _feature_matrix(
self, df: pd.DataFrame, include_categorical: bool
) -> Tuple[np.ndarray, np.ndarray, List[str]]:
numeric_cols = self._numeric_columns(df)
X_num = df[numeric_cols].apply(pd.to_numeric, errors="coerce")
for col in numeric_cols:
med = pd.to_numeric(X_num[col], errors="coerce").median()
X_num[col] = pd.to_numeric(X_num[col], errors="coerce").fillna(med)
parts = [X_num]
feat_names = list(X_num.columns)
if include_categorical and self.cat_cols:
cat_cols = [c for c in self.cat_cols if c in df.columns]
if cat_cols:
df_cats = pd.get_dummies(
df[cat_cols].astype("category"), drop_first=False, prefix=cat_cols
)
parts.append(df_cats)
feat_names.extend(list(df_cats.columns))
X = pd.concat(parts, axis=1).values.astype(np.float32)
labels = df[self.label_col]
y_all, keep_mask = self._select_binary_labels(labels)
y = y_all[keep_mask]
X = X[keep_mask]
return X, y.astype(int), feat_names
def univariate_roc(
self, merge: bool = False, include_categorical: bool = False
) -> pd.DataFrame:
clinical = self._build_clinical()
df = clinical.df.copy()
labels = df[self.label_col]
y_all, keep_mask = self._select_binary_labels(labels)
if self.debug:
for col in ("IOP_raw", "IOP_corr"):
if col not in df.columns:
print(f"[debug] {col} missing from df")
continue
s = pd.to_numeric(df[col], errors="coerce")
print(
f"[debug] {col}: non-null={int(s.notna().sum())}, unique={int(s.nunique(dropna=True))}"
)
plot_dir = self.output_dir / "papila_univariate_roc" / "plots"
plot_dir.mkdir(parents=True, exist_ok=True)
rows = []
curves = []
numeric_cols = self._numeric_columns(df)
for col in numeric_cols:
series = pd.to_numeric(df[col], errors="coerce")
mask = keep_mask & series.notna().values
y = y_all[mask]
scores = series.values[mask].astype(float)
if y.size < 2 or np.unique(y).size < 2:
continue
if np.nanmin(scores) == np.nanmax(scores):
continue
fpr, tpr, thresholds, auc_val = self._compute_roc(y, scores)
thr, best_tpr, best_fpr = self._best_threshold(fpr, tpr, thresholds)
direction = "high" if auc_val >= 0.5 else "low"
title = f"{col} (n={y.size}, direction={direction})"
if merge:
out_path = plot_dir / f"roc_{self._sanitize(col)}.png"
self._plot_per_feature(fpr, tpr, auc_val, title, out_path)
curves.append((col, fpr, tpr, auc_val))
rows.append(
{
"feature": col,
"kind": "numeric",
"n": int(y.size),
"auc": auc_val,
"direction": direction,
"best_threshold": thr,
"best_tpr": best_tpr,
"best_fpr": best_fpr,
"best_specificity": 1.0 - best_fpr,
}
)
if include_categorical:
cat_cols_use = [c for c in self.cat_cols if c not in self.exclude_cols]
for name, ind in self._iter_categorical(df, cat_cols_use):
mask = keep_mask & ind.notna().values
y = y_all[mask]
scores = ind.values[mask].astype(float)
if y.size < 2 or np.unique(y).size < 2:
continue
if np.nanmin(scores) == np.nanmax(scores):
continue
fpr, tpr, thresholds, auc_val = self._compute_roc(y, scores)
thr, best_tpr, best_fpr = self._best_threshold(fpr, tpr, thresholds)
direction = "high" if auc_val >= 0.5 else "low"
title = f"{name} (n={y.size}, direction={direction})"
if merge:
out_path = plot_dir / f"roc_{self._sanitize(name)}.png"
self._plot_per_feature(fpr, tpr, auc_val, title, out_path)
curves.append((name, fpr, tpr, auc_val))
rows.append(
{
"feature": name,
"kind": "categorical",
"n": int(y.size),
"auc": auc_val,
"direction": direction,
"best_threshold": thr,
"best_tpr": best_tpr,
"best_fpr": best_fpr,
"best_specificity": 1.0 - best_fpr,
}
)
if not rows:
raise SystemExit(
"No valid features produced ROC curves. Check labels and feature columns."
)
overlay_path = plot_dir / "roc_overlay.png"
if not merge:
self._plot_overlay(curves, "Univariate ROC curves", overlay_path)
out_df = pd.DataFrame(rows).sort_values(by="auc", ascending=False)
out_csv = self.output_dir / "papila_univariate_roc" / "summary.csv"
out_csv.parent.mkdir(parents=True, exist_ok=True)
out_df.to_csv(out_csv, index=False)
return out_df
def random_forest(
self,
include_categorical: bool = True,
n_estimators: int = 500,
max_depth: int | None = None,
min_samples_leaf: int = 1,
max_features: str | None = "sqrt",
class_weight: str | None = "balanced",
max_samples: float | None = None,
random_state: int = 42,
top_n: int = 25,
n_splits: int = 5,
drop_missing: bool = False,
nerf: bool = False,
drop_age: bool = False,
) -> pd.DataFrame:
clinical = self._build_clinical()
df = clinical.df.copy()
original_exclude = set(self.exclude_cols)
if nerf:
self.exclude_cols = set(self.exclude_cols)
drop_missing = True
if drop_age:
self.exclude_cols = set(self.exclude_cols) | {"Age"}
if drop_missing:
numeric_cols = self._numeric_columns(df)
df = df.dropna(subset=numeric_cols)
X, y, feat_names = self._feature_matrix(
df, include_categorical=include_categorical
)
if X.size == 0 or np.unique(y).size < 2:
raise SystemExit(
"Not enough data after filtering labels for Random Forest."
)
if nerf:
n_estimators = 200
max_depth = 5
min_samples_leaf = 5
max_features = "sqrt"
class_weight = None
max_samples = 0.7
self.exclude_cols = original_exclude
clf = RandomForestClassifier(
n_estimators=n_estimators,
max_depth=max_depth,
min_samples_leaf=min_samples_leaf,
max_features=max_features,
class_weight=class_weight,
max_samples=max_samples,
random_state=random_state,
n_jobs=-1,
)
clf.fit(X, y)
importances = clf.feature_importances_.astype(float)
rows = []
for name, val in zip(feat_names, importances):
rows.append({"feature": name, "importance": float(val)})
out_df = pd.DataFrame(rows).sort_values(by="importance", ascending=False)
out_dir = self.output_dir / (
"papila_random_forest_nerfed" if nerf else "papila_random_forest"
)
out_dir.mkdir(parents=True, exist_ok=True)
out_df.to_csv(out_dir / "feature_importance.csv", index=False)
top_df = out_df.head(top_n)
fig, ax = plt.subplots(figsize=(7, 6))
ax.barh(top_df["feature"], top_df["importance"], color="steelblue")
ax.invert_yaxis()
ax.set_xlabel("Importance (Gini)")
ax.set_title(f"Random Forest Feature Importance")
fig.tight_layout()
fig.savefig(out_dir / "feature_importance_top.png", dpi=170)
plt.close(fig)
if self.debug:
age_rows = out_df[out_df["feature"] == "Age"]
if not age_rows.empty:
age_imp = float(age_rows["importance"].iloc[0])
print(f"[debug] RF importance Age = {age_imp:.4f}")
y_oof, scores_oof = self._oof_scores(clf, X, y, n_splits, random_state)
fpr, tpr, _, auc_val = self._compute_roc(y_oof, scores_oof)
curves = self._cv_roc_curves(clf, X, y, n_splits, random_state)
self._plot_mean_roc(
curves,
"Random Forest ROC (mean ± SD)",
out_dir / "roc_mean.png",
)
return out_df
def _cv_binary_metrics(
self,
model,
X: np.ndarray,
y: np.ndarray,
n_splits: int,
random_state: int,
) -> pd.DataFrame:
skf = StratifiedKFold(
n_splits=n_splits, shuffle=True, random_state=random_state
)
rows = []
for fold, (train_idx, test_idx) in enumerate(skf.split(X, y), start=1):
X_train, X_test = X[train_idx], X[test_idx]
y_train, y_test = y[train_idx], y[test_idx]
if np.unique(y_train).size < 2 or np.unique(y_test).size < 2:
continue
model.fit(X_train, y_train)
if hasattr(model, "predict_proba"):
scores = model.predict_proba(X_test)[:, 1]
elif hasattr(model, "decision_function"):
scores = model.decision_function(X_test)
else:
scores = model.predict(X_test)
preds = model.predict(X_test)
auc_val = float(roc_auc_score(y_test, scores))
acc_val = float(accuracy_score(y_test, preds))
rows.append(
{
"fold": int(fold),
"n": int(len(y_test)),
"auc": auc_val,
"acc": acc_val,
}
)
return pd.DataFrame(rows)
def svm(
self,
include_categorical: bool = True,
kernel: str = "rbf",
C: float = 1.0,
gamma: str = "scale",
n_splits: int = 5,
random_state: int = 42,
) -> pd.DataFrame:
clinical = self._build_clinical()
df = clinical.df.copy()
X, y, feat_names = self._feature_matrix(
df, include_categorical=include_categorical
)
if X.size == 0 or np.unique(y).size < 2:
raise SystemExit("Not enough data after filtering labels for SVM.")
model = Pipeline(
[
("scale", StandardScaler()),
(
"svm",
SVC(
kernel=kernel,
C=C,
gamma=gamma,
probability=True,
class_weight="balanced",
random_state=random_state,
),
),
]
)
fold_df = self._cv_binary_metrics(model, X, y, n_splits, random_state)
if fold_df.empty:
raise SystemExit("SVM produced no valid folds (check class balance).")
y_oof, scores_oof = self._oof_scores(model, X, y, n_splits, random_state)
fpr, tpr, _, auc_val = self._compute_roc(y_oof, scores_oof)
curves = self._cv_roc_curves(model, X, y, n_splits, random_state)
summary = pd.DataFrame(
[
{
"metric": "auc",
"mean": float(fold_df["auc"].mean()),
"std": float(fold_df["auc"].std(ddof=0)),
"oof_auc": float(auc_val),
},
{
"metric": "acc",
"mean": float(fold_df["acc"].mean()),
"std": float(fold_df["acc"].std(ddof=0)),
},
]
)
out_dir = self.output_dir / "papila_svm"
out_dir.mkdir(parents=True, exist_ok=True)
fold_df.to_csv(out_dir / "fold_metrics.csv", index=False)
summary.to_csv(out_dir / "summary.csv", index=False)
self._plot_mean_roc(
curves,
"SVM ROC (mean ± SD)",
out_dir / "roc_mean.png",
)
return fold_df
def knn(
self,
include_categorical: bool = True,
n_neighbors: int = 5,
weights: str = "distance",
n_splits: int = 5,
random_state: int = 42,
) -> pd.DataFrame:
clinical = self._build_clinical()
df = clinical.df.copy()
X, y, feat_names = self._feature_matrix(
df, include_categorical=include_categorical
)
if X.size == 0 or np.unique(y).size < 2:
raise SystemExit("Not enough data after filtering labels for KNN.")
model = Pipeline(
[
("scale", StandardScaler()),
("knn", KNeighborsClassifier(n_neighbors=n_neighbors, weights=weights)),
]
)
fold_df = self._cv_binary_metrics(model, X, y, n_splits, random_state)
if fold_df.empty:
raise SystemExit("KNN produced no valid folds (check class balance).")
y_oof, scores_oof = self._oof_scores(model, X, y, n_splits, random_state)
fpr, tpr, _, auc_val = self._compute_roc(y_oof, scores_oof)
curves = self._cv_roc_curves(model, X, y, n_splits, random_state)
summary = pd.DataFrame(
[
{
"metric": "auc",
"mean": float(fold_df["auc"].mean()),
"std": float(fold_df["auc"].std(ddof=0)),
"oof_auc": float(auc_val),
},
{
"metric": "acc",
"mean": float(fold_df["acc"].mean()),
"std": float(fold_df["acc"].std(ddof=0)),
},
]
)
out_dir = self.output_dir / "papila_knn"
out_dir.mkdir(parents=True, exist_ok=True)
fold_df.to_csv(out_dir / "fold_metrics.csv", index=False)
summary.to_csv(out_dir / "summary.csv", index=False)
self._plot_mean_roc(
curves,
"KNN ROC (mean ± SD)",
out_dir / "roc_mean.png",
)
return fold_df
def logistic_regression(
self,
include_categorical: bool = True,
C: float = 1.0,
max_iter: int = 1000,
n_splits: int = 5,
random_state: int = 42,
drop_age: bool = False,
nerf: bool = False,
) -> pd.DataFrame:
clinical = self._build_clinical()
df = clinical.df.copy()
original_exclude = set(self.exclude_cols)
if drop_age:
self.exclude_cols = set(self.exclude_cols) | {"Age"}
X, y, feat_names = self._feature_matrix(
df, include_categorical=include_categorical
)
self.exclude_cols = original_exclude
if X.size == 0 or np.unique(y).size < 2:
raise SystemExit(
"Not enough data after filtering labels for Logistic Regression."
)
class_weight = "balanced"
penalty = "l2"
solver = "lbfgs"
if nerf:
C = 0.05
class_weight = None
penalty = "l1"
solver = "liblinear"
model = Pipeline(
[
("scale", StandardScaler()),
(
"logreg",
LogisticRegression(
C=C,
max_iter=max_iter,
class_weight=class_weight,
penalty=penalty,
solver=solver,
),
),
]
)
fold_df = self._cv_binary_metrics(model, X, y, n_splits, random_state)
if fold_df.empty:
raise SystemExit(
"Logistic Regression produced no valid folds (check class balance)."
)
y_oof, scores_oof = self._oof_scores(model, X, y, n_splits, random_state)
fpr, tpr, _, auc_val = self._compute_roc(y_oof, scores_oof)
curves = self._cv_roc_curves(model, X, y, n_splits, random_state)
summary = pd.DataFrame(
[
{
"metric": "auc",
"mean": float(fold_df["auc"].mean()),
"std": float(fold_df["auc"].std(ddof=0)),
"oof_auc": float(auc_val),
},
{
"metric": "acc",
"mean": float(fold_df["acc"].mean()),
"std": float(fold_df["acc"].std(ddof=0)),
},
]
)
out_dir = self.output_dir / "papila_logistic_regression"
out_dir.mkdir(parents=True, exist_ok=True)
fold_df.to_csv(out_dir / "fold_metrics.csv", index=False)
summary.to_csv(out_dir / "summary.csv", index=False)
self._plot_mean_roc(
curves,
"Logistic Regression ROC (mean ± SD)",
out_dir / "roc_mean.png",
)
return fold_df
ba = basic_analytics()
roc_df = ba.univariate_roc(merge=False, include_categorical=False)
rf_df = ba.random_forest(include_categorical=True, nerf=False)
svm_df = ba.svm(include_categorical=True)
knn_df = ba.knn(include_categorical=True)
lr_df = ba.logistic_regression(include_categorical=True, nerf=True)
clinical = ba._build_clinical()
clinical.df["Diagnosis"].value_counts()
scripts/basic_analysis/cnn_logits_rf_cv.py
Executable
+327
View File
@@ -0,0 +1,327 @@
#!/usr/bin/env python3
"""Train a CNN (resnet50 backbone), extract logits, and train RF on logits+metadata with 5-fold CV."""
from __future__ import annotations
import random
from pathlib import Path
import sys
from typing import Dict, List, Tuple
import numpy as np
import pandas as pd
from PIL import Image
import torch
from torch import nn
from torch.utils.data import DataLoader, Dataset
from torchvision import transforms
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, roc_auc_score
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes import build_papila_clinical
from classes.backbones import BACKBONES, load_backbone_weights
# ---------------------------
# Config (edit in IDE)
# ---------------------------
IMAGE_DIR = "Papila/FundusImages"
CLINICAL_DIR = "Papila/ClinicalData"
LABEL_COL = "Diagnosis"
CAT_COLS = ["Gender", "Phakic/Pseudophakic"]
EVAL_MODE = "binary" # "binary" or "multiclass"
N_SPLITS = 5
FOLD_SEED = 42
HOLDOUT_SEED = 123
HOLDOUT_PATIENTS_PER_CLASS = 6
BACKBONE_NAME = "resnet50"
BATCH_SIZE = 8
EPOCHS = 40
LR = 1e-4
WEIGHT_DECAY = 1e-5
RF_TREES = 500
RF_MAX_DEPTH = None
RF_MIN_SAMPLES_LEAF = 1
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
OUTPUT_DIR = Path("analysis_data/basic_analysis/cnn_logits_rf_cv")
PRINT_EPOCH_REPORT = True
EPOCH_REPORT_EVERY = 1
class PapilaImageDataset(Dataset):
def __init__(
self, clinical, df: pd.DataFrame, label_col: str, img_transform
) -> None:
self.clinical = clinical
self.df = df.reset_index(drop=True)
self.label_col = label_col
self.img_transform = img_transform
def __len__(self) -> int:
return len(self.df)
def __getitem__(self, idx: int):
row = self.df.iloc[idx]
img_path = self.clinical.get_image_path(row)
image = Image.open(img_path).convert("RGB")
x_img = self.img_transform(image)
y = int(row[self.label_col])
x_md = self.clinical.vectorize_row(row).astype(np.float32)
return x_img, y, x_md
class CNNHead(nn.Module):
def __init__(self, backbone_name: str, num_classes: int) -> None:
super().__init__()
spec = BACKBONES[backbone_name]
backbone = spec.ctor(weights=spec.weights_default)
if backbone_name.startswith("refuge"):
load_backbone_weights(backbone_name, backbone)
out_dim, backbone = spec.strip(backbone)
self.backbone = backbone
self.head = nn.Linear(out_dim, num_classes)
def forward(self, x: torch.Tensor) -> torch.Tensor:
feats = self.backbone(x)
return self.head(feats)
def _set_seed(seed: int) -> None:
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
def _auc_score(y_true: np.ndarray, probs: np.ndarray, num_classes: int) -> float:
try:
if num_classes == 2:
return float(roc_auc_score(y_true, probs[:, 1]))
return float(roc_auc_score(y_true, probs, multi_class="ovr", average="macro"))
except Exception:
return float("nan")
def _prepare_clinical() -> Tuple[object, pd.DataFrame]:
clinical = build_papila_clinical(
image_dir=IMAGE_DIR,
clinical_dir=CLINICAL_DIR,
label_col=LABEL_COL,
cat_cols=CAT_COLS,
n_splits=N_SPLITS,
random_seed=FOLD_SEED,
)
df = clinical.df.copy()
if EVAL_MODE == "binary":
df = df[df[LABEL_COL].isin([0, 1])].reset_index(drop=True)
return clinical, df
def _split_holdout_by_patient(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:
rng = np.random.default_rng(HOLDOUT_SEED)
patient_label = (
df.groupby("Patient ID")[LABEL_COL]
.agg(lambda s: int(s.mode().iloc[0]))
.reset_index()
)
holdout_patients = []
for lbl, grp in patient_label.groupby(LABEL_COL):
candidates = grp["Patient ID"].to_numpy()
n = min(HOLDOUT_PATIENTS_PER_CLASS, len(candidates))
if n <= 0:
continue
selected = rng.choice(candidates, size=n, replace=False)
holdout_patients.extend(selected.tolist())
holdout_patients = sorted(set(holdout_patients))
holdout_df = df[df["Patient ID"].isin(holdout_patients)].reset_index(drop=True)
train_df = df[~df["Patient ID"].isin(holdout_patients)].reset_index(drop=True)
return train_df, holdout_df
def _rebuild_clinical_from_df(clinical, df: pd.DataFrame) -> object:
clinical.frames = [df.copy()]
clinical.df = df.copy()
clinical._infer_or_validate_feature_types()
clinical._compute_numeric_stats()
clinical._build_cat_maps()
clinical._compute_feature_dim()
clinical._build_kfold_indices()
return clinical
def _train_cnn(
model: nn.Module, loader: DataLoader, num_classes: int, fold: int
) -> None:
model.train()
optimizer = torch.optim.Adam(model.parameters(), lr=LR, weight_decay=WEIGHT_DECAY)
criterion = nn.CrossEntropyLoss()
for epoch in range(EPOCHS):
running_loss = 0.0
correct = 0
total = 0
for x_img, y, _x_md in loader:
x_img = x_img.to(DEVICE)
y = y.to(DEVICE)
optimizer.zero_grad()
logits = model(x_img)
loss = criterion(logits, y)
loss.backward()
optimizer.step()
running_loss += float(loss.item()) * int(y.size(0))
pred = torch.argmax(logits, dim=1)
correct += int((pred == y).sum().item())
total += int(y.size(0))
if PRINT_EPOCH_REPORT and ((epoch + 1) % EPOCH_REPORT_EVERY == 0):
avg_loss = running_loss / max(total, 1)
train_acc = correct / max(total, 1)
print(
f"[fold {fold + 1}/{N_SPLITS}] epoch {epoch + 1}/{EPOCHS} "
f"train_loss={avg_loss:.4f} train_acc={train_acc:.4f}"
)
def _infer_logits(
model: nn.Module, loader: DataLoader
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
model.eval()
logits_all, probs_all, y_all, md_all = [], [], [], []
with torch.no_grad():
for x_img, y, x_md in loader:
x_img = x_img.to(DEVICE)
logits = model(x_img).cpu().numpy()
probs = torch.softmax(torch.from_numpy(logits), dim=1).numpy()
logits_all.append(logits)
probs_all.append(probs)
y_all.append(y.numpy())
md_all.append(x_md.numpy())
return (
np.concatenate(y_all, axis=0),
np.concatenate(logits_all, axis=0),
np.concatenate(md_all, axis=0),
)
def main() -> None:
_set_seed(FOLD_SEED)
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
num_classes = 2 if EVAL_MODE == "binary" else 3
train_tf = transforms.Compose(
[
transforms.Resize((224, 224)),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
]
)
eval_tf = transforms.Compose(
[
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
]
)
clinical, df = _prepare_clinical()
train_df, holdout_df = _split_holdout_by_patient(df)
clinical = _rebuild_clinical_from_df(clinical, train_df)
holdout_df.to_csv(OUTPUT_DIR / "holdout_patients.csv", index=False)
rows: List[Dict[str, object]] = []
holdout_rows: List[Dict[str, object]] = []
for fold in range(N_SPLITS):
print(f"\n[info] Starting fold {fold + 1}/{N_SPLITS}")
fold_train_df, fold_val_df = clinical.get_split_dfs(fold)
ds_train = PapilaImageDataset(clinical, fold_train_df, LABEL_COL, train_tf)
ds_val = PapilaImageDataset(clinical, fold_val_df, LABEL_COL, eval_tf)
ds_holdout = PapilaImageDataset(clinical, holdout_df, LABEL_COL, eval_tf)
dl_train = DataLoader(
ds_train, batch_size=BATCH_SIZE, shuffle=True, num_workers=0
)
dl_val = DataLoader(ds_val, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)
dl_holdout = DataLoader(
ds_holdout, batch_size=BATCH_SIZE, shuffle=False, num_workers=0
)
model = CNNHead(BACKBONE_NAME, num_classes=num_classes).to(DEVICE)
_train_cnn(model, dl_train, num_classes=num_classes, fold=fold)
y_tr, log_tr, md_tr = _infer_logits(
model,
DataLoader(ds_train, batch_size=BATCH_SIZE, shuffle=False, num_workers=0),
)
y_va, log_va, md_va = _infer_logits(model, dl_val)
y_ho, log_ho, md_ho = _infer_logits(model, dl_holdout)
np.save(OUTPUT_DIR / f"fold{fold}_train_logits.npy", log_tr)
np.save(OUTPUT_DIR / f"fold{fold}_val_logits.npy", log_va)
np.save(OUTPUT_DIR / f"fold{fold}_holdout_logits.npy", log_ho)
X_tr = np.concatenate([log_tr, md_tr], axis=1)
X_va = np.concatenate([log_va, md_va], axis=1)
X_ho = np.concatenate([log_ho, md_ho], axis=1)
rf = RandomForestClassifier(
n_estimators=RF_TREES,
max_depth=RF_MAX_DEPTH,
min_samples_leaf=RF_MIN_SAMPLES_LEAF,
class_weight="balanced",
random_state=FOLD_SEED + fold,
n_jobs=-1,
)
rf.fit(X_tr, y_tr)
p_va = rf.predict_proba(X_va)
p_ho = rf.predict_proba(X_ho)
pred_va = np.argmax(p_va, axis=1)
pred_ho = np.argmax(p_ho, axis=1)
rows.append(
{
"fold": fold,
"val_acc": float(accuracy_score(y_va, pred_va)),
"val_auc": _auc_score(y_va, p_va, num_classes),
"n_val": int(len(y_va)),
}
)
holdout_rows.append(
{
"fold": fold,
"holdout_acc": float(accuracy_score(y_ho, pred_ho)),
"holdout_auc": _auc_score(y_ho, p_ho, num_classes),
"n_holdout": int(len(y_ho)),
}
)
print(
f"[info] Fold {fold + 1} RF: val_acc={rows[-1]['val_acc']:.4f} val_auc={rows[-1]['val_auc']:.4f} "
f"| holdout_acc={holdout_rows[-1]['holdout_acc']:.4f} holdout_auc={holdout_rows[-1]['holdout_auc']:.4f}"
)
fold_df = pd.DataFrame(rows)
holdout_df = pd.DataFrame(holdout_rows)
fold_df.to_csv(OUTPUT_DIR / "rf_val_metrics.csv", index=False)
holdout_df.to_csv(OUTPUT_DIR / "rf_holdout_metrics.csv", index=False)
print("\nRF validation metrics:")
print(fold_df.to_string(index=False, float_format=lambda x: f"{x:.4f}"))
print("\nRF holdout metrics:")
print(holdout_df.to_string(index=False, float_format=lambda x: f"{x:.4f}"))
print(
f"\nMeans: val_acc={fold_df['val_acc'].mean():.4f}, val_auc={fold_df['val_auc'].mean():.4f}, "
f"holdout_acc={holdout_df['holdout_acc'].mean():.4f}, holdout_auc={holdout_df['holdout_auc'].mean():.4f}"
)
print(f"\nSaved outputs to: {OUTPUT_DIR}")
if __name__ == "__main__":
main()
scripts/basic_analysis/compare_dual_eye_towers.py
+1338
View File
File diff suppressed because it is too large Load Diff
scripts/basic_analysis/compare_hypertower_modes.py
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Thin CLI wrapper that runs V2 hypertower modes sequentially."""
from __future__ import annotations
from pathlib import Path
import sys
import argparse
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.v2.v2_hypertower import build_parser, run_mode
def parse_args():
ap = argparse.ArgumentParser(
description="Run selected eval/tower mode combinations sequentially."
)
ap.add_argument(
"--eval-modes",
nargs="+",
choices=["binary", "multiclass"],
default=["binary", "multiclass"],
)
ap.add_argument(
"--tower-modes",
nargs="+",
choices=["single", "ensemble", "bilateral", "classic"],
default=["single", "ensemble", "bilateral"],
)
return ap.parse_known_args()
def main():
seq_args, remaining = parse_args()
base_parser = build_parser()
for eval_mode in seq_args.eval_modes:
for tower_mode in seq_args.tower_modes:
tower_mode = "single" if tower_mode == "classic" else tower_mode
cli = list(remaining) + ["--eval-mode", eval_mode, "--tower-mode", tower_mode]
args = base_parser.parse_args(cli)
run_mode(args)
if __name__ == "__main__":
main()
scripts/basic_analysis/compare_siamese_tower.py
+725
View File
@@ -0,0 +1,725 @@
#!/usr/bin/env python3
"""
Compare single-eye OD baseline vs SiameseImageTower bilateral model.
Key differences from compare_dual_eye_towers.py:
- Uses SiameseImageTower (shared backbone, f_mean + f_delta output).
- Reports BEST-epoch val metrics per fold (not final-epoch), with the
corresponding holdout metrics snapped at the same checkpoint.
- Both models are always evaluated on patient-level samples (matched n).
- Optionally includes two-single-merge as a second reference point.
"""
from __future__ import annotations
import argparse
import copy
import csv
import json
import random
import sys
import time
from dataclasses import dataclass, field
from pathlib import Path
from types import SimpleNamespace
from typing import Optional
import numpy as np
import torch
import torch.nn.functional as F
from sklearn.metrics import roc_auc_score
from torch import nn
from torch.utils.data import DataLoader
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.v2 import (
ImageTower,
PatientFirstSplitManager,
SiameseImageTower,
SlotDataset,
build_papila_data,
build_papila_profile,
slot_collate,
)
# ---------------------------------------------------------------------------
# Reproducibility
# ---------------------------------------------------------------------------
def seed_everything(seed: int) -> None:
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
# ---------------------------------------------------------------------------
# Models
# ---------------------------------------------------------------------------
class BaselineCNN(nn.Module):
"""Single-eye (OD) image tower with a linear head."""
def __init__(self, *, backbone: str, freeze_ratio: float, num_classes: int, augment: bool):
super().__init__()
self.tower = ImageTower(
backbone=backbone,
freeze_ratio=freeze_ratio,
augment=augment,
use_se=False,
)
self.head = nn.Linear(self.tower.out_dim, num_classes)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.head(self.tower(x))
class SiameseCNN(nn.Module):
"""
Bilateral image model using SiameseImageTower.
forward(x_od, x_os) -> logits
"""
def __init__(self, *, backbone: str, freeze_ratio: float, num_classes: int, augment: bool):
super().__init__()
self.tower = SiameseImageTower(
backbone=backbone,
freeze_ratio=freeze_ratio,
augment=augment,
use_se=False,
)
self.head = nn.Linear(self.tower.out_dim, num_classes)
def forward(self, x_od: torch.Tensor, x_os: torch.Tensor) -> torch.Tensor:
return self.head(self.tower(x_od, x_os))
# ---------------------------------------------------------------------------
# Data helpers
# ---------------------------------------------------------------------------
def filter_od_samples(patient_samples: list[dict]) -> list[dict]:
"""Extract patient-level OD-only samples (image_1 = OD)."""
out = []
for s in patient_samples:
if s.get("image_1") is not None and s.get("label_1") is not None:
out.append({"id_1": s.get("id_1"), "image_1": s["image_1"], "label_1": s["label_1"]})
return out
def filter_bilateral_samples(patient_samples: list[dict]) -> list[dict]:
return [
s for s in patient_samples
if s.get("image_1") is not None
and s.get("image_2") is not None
and s.get("label_1") is not None
]
def make_loader(samples, slots, *, image_transform, batch_size, shuffle, num_workers) -> DataLoader:
ds = SlotDataset(samples, slots, image_transform=image_transform)
return DataLoader(
ds,
batch_size=batch_size,
shuffle=shuffle,
num_workers=num_workers,
collate_fn=slot_collate,
)
def to_label_tensor(labels, device: torch.device) -> torch.Tensor:
if torch.is_tensor(labels):
return labels.to(device=device, dtype=torch.long)
return torch.as_tensor(labels, dtype=torch.long, device=device)
def _drop_mixed_label_patients(df, *, patient_col: str, label_col: str):
import pandas as pd
per_patient = (
df.groupby(patient_col)[label_col]
.agg(lambda s: set(pd.to_numeric(s, errors="coerce").dropna().astype(int).tolist()))
)
mixed = [pid for pid, labels in per_patient.items() if len(labels) > 1]
if not mixed:
return df, []
return df[~df[patient_col].isin(mixed)].reset_index(drop=True), mixed
# ---------------------------------------------------------------------------
# Score helpers
# ---------------------------------------------------------------------------
def _score(y_true_chunks, y_prob_chunks, num_classes: int):
if not y_true_chunks:
return float("nan"), float("nan"), 0
y = np.concatenate(y_true_chunks)
p = np.concatenate(y_prob_chunks)
acc = float((p.argmax(1) == y).mean())
try:
auc = (
float(roc_auc_score(y, p[:, 1]))
if num_classes == 2
else float(roc_auc_score(y, p, multi_class="ovr", average="macro"))
)
except Exception:
auc = float("nan")
return acc, auc, int(len(y))
# ---------------------------------------------------------------------------
# Train / evaluate
# ---------------------------------------------------------------------------
def train_baseline_epoch(model, loader, opt, device):
model.train()
total_loss = total_correct = total_n = 0
for batch in loader:
x = batch.get("image_1")
y = batch.get("label_1")
if not torch.is_tensor(x):
continue
y = to_label_tensor(y, device)
x = x.to(device)
logits = model(x)
loss = F.cross_entropy(logits, y)
opt.zero_grad()
loss.backward()
opt.step()
bs = y.shape[0]
total_loss += float(loss.item()) * bs
total_correct += int((logits.argmax(1) == y).sum())
total_n += bs
return (total_loss / total_n if total_n else float("nan"),
total_correct / total_n if total_n else float("nan"))
def train_siamese_epoch(model, loader, opt, device):
model.train()
total_loss = total_correct = total_n = 0
for batch in loader:
x1 = batch.get("image_1")
x2 = batch.get("image_2")
y = batch.get("label_1")
if not torch.is_tensor(x1) or not torch.is_tensor(x2):
continue
y = to_label_tensor(y, device)
logits = model(x1.to(device), x2.to(device))
loss = F.cross_entropy(logits, y)
opt.zero_grad()
loss.backward()
opt.step()
bs = y.shape[0]
total_loss += float(loss.item()) * bs
total_correct += int((logits.argmax(1) == y).sum())
total_n += bs
return (total_loss / total_n if total_n else float("nan"),
total_correct / total_n if total_n else float("nan"))
def evaluate_baseline(model, loader, device, num_classes):
model.eval()
y_true, y_prob = [], []
with torch.no_grad():
for batch in loader:
x = batch.get("image_1")
y = batch.get("label_1")
if not torch.is_tensor(x):
continue
y_t = to_label_tensor(y, device)
p = F.softmax(model(x.to(device)), dim=1).cpu().numpy()
y_true.append(y_t.cpu().numpy())
y_prob.append(p)
return _score(y_true, y_prob, num_classes)
def evaluate_siamese(model, loader, device, num_classes):
model.eval()
y_true, y_prob = [], []
with torch.no_grad():
for batch in loader:
x1 = batch.get("image_1")
x2 = batch.get("image_2")
y = batch.get("label_1")
if not torch.is_tensor(x1) or not torch.is_tensor(x2):
continue
y_t = to_label_tensor(y, device)
p = F.softmax(model(x1.to(device), x2.to(device)), dim=1).cpu().numpy()
y_true.append(y_t.cpu().numpy())
y_prob.append(p)
return _score(y_true, y_prob, num_classes)
# ---------------------------------------------------------------------------
# Result dataclass
# ---------------------------------------------------------------------------
@dataclass
class FoldResult:
mode: str
fold: int
# Best-epoch validation metrics
best_epoch: int
baseline_best_val_auc: float
baseline_best_val_acc: float
siamese_best_val_auc: float
siamese_best_val_acc: float
# Holdout metrics at the respective best-epoch checkpoint
baseline_holdout_auc: float
baseline_holdout_acc: float
baseline_holdout_n: int
siamese_holdout_auc: float
siamese_holdout_acc: float
siamese_holdout_n: int
# Sample sizes
baseline_n: int
siamese_n: int
def _nan() -> float:
return float("nan")
# ---------------------------------------------------------------------------
# Main fold runner
# ---------------------------------------------------------------------------
def run_fold(
fold: int,
split,
mode: str,
args,
device: torch.device,
data,
num_classes: int,
profile_od,
profile_patient,
fold_dir: Path,
) -> FoldResult:
holdout_df = split.holdout
# ---- build samples ----
bilat_train = filter_bilateral_samples(profile_patient.build_samples(df=split.train, clinical=data))
bilat_val = filter_bilateral_samples(profile_patient.build_samples(df=split.val, clinical=data))
od_train = filter_od_samples(bilat_train)
od_val = filter_od_samples(bilat_val)
bilat_holdout = []
od_holdout = []
if holdout_df is not None and not holdout_df.empty:
bilat_holdout = filter_bilateral_samples(profile_patient.build_samples(df=holdout_df, clinical=data))
od_holdout = filter_od_samples(bilat_holdout)
# ---- models ----
baseline = BaselineCNN(
backbone=args.backbone, freeze_ratio=args.freeze_ratio,
num_classes=num_classes, augment=args.augment,
).to(device)
siamese = SiameseCNN(
backbone=args.backbone, freeze_ratio=args.freeze_ratio,
num_classes=num_classes, augment=args.augment,
).to(device)
slots_od = profile_od.slot_descriptors()
slots_patient = profile_patient.slot_descriptors()
# ---- loaders ----
loader_kw = dict(batch_size=args.batch_size, num_workers=args.num_workers)
train_base = make_loader(od_train, slots_od, image_transform=baseline.tower.transform, shuffle=True, **loader_kw)
val_base = make_loader(od_val, slots_od, image_transform=baseline.tower.transform, shuffle=False, **loader_kw)
train_siam = make_loader(bilat_train, slots_patient, image_transform=siamese.tower.transform, shuffle=True, **loader_kw)
val_siam = make_loader(bilat_val, slots_patient, image_transform=siamese.tower.transform, shuffle=False, **loader_kw)
ho_base = (
make_loader(od_holdout, slots_od, image_transform=baseline.tower.transform, shuffle=False, **loader_kw)
if od_holdout else None
)
ho_siam = (
make_loader(bilat_holdout, slots_patient, image_transform=siamese.tower.transform, shuffle=False, **loader_kw)
if bilat_holdout else None
)
opt_base = torch.optim.Adam(baseline.parameters(), lr=args.lr)
opt_siam = torch.optim.Adam(siamese.parameters(), lr=args.lr)
# ---- epoch log ----
epoch_log_path = fold_dir / "epoch_log.csv"
epoch_fields = [
"fold", "epoch",
"base_train_loss", "base_train_acc",
"base_val_auc", "base_val_acc", "base_val_n",
"siam_train_loss", "siam_train_acc",
"siam_val_auc", "siam_val_acc", "siam_val_n",
"ho_base_auc", "ho_base_acc", "ho_base_n",
"ho_siam_auc", "ho_siam_acc", "ho_siam_n",
]
epoch_fp = epoch_log_path.open("w", newline="", encoding="utf-8")
epoch_writer = csv.DictWriter(epoch_fp, fieldnames=epoch_fields)
epoch_writer.writeheader()
def _f(v):
return None if (v is None or (isinstance(v, float) and np.isnan(v))) else round(float(v), 6)
# ---- best-epoch tracking ----
best_base_auc = -1.0
best_siam_auc = -1.0
best_base_state: Optional[dict] = None
best_siam_state: Optional[dict] = None
best_base_val_acc = _nan()
best_siam_val_acc = _nan()
# Holdout metrics snapped at best-val checkpoint
snap_ho_base_auc = _nan()
snap_ho_base_acc = _nan()
snap_ho_base_n = 0
snap_ho_siam_auc = _nan()
snap_ho_siam_acc = _nan()
snap_ho_siam_n = 0
best_epoch = 0
print(
f" [fold {fold+1}] training {args.epochs} epochs | "
f"baseline n_train={len(od_train)} n_val={len(od_val)} | "
f"siamese n_train={len(bilat_train)} n_val={len(bilat_val)}",
flush=True,
)
for epoch in range(args.epochs):
bl_loss, bl_acc = train_baseline_epoch(baseline, train_base, opt_base, device)
si_loss, si_acc = train_siamese_epoch(siamese, train_siam, opt_siam, device)
b_val_acc, b_val_auc, b_val_n = evaluate_baseline(baseline, val_base, device, num_classes)
s_val_acc, s_val_auc, s_val_n = evaluate_siamese( siamese, val_siam, device, num_classes)
# Holdout at this epoch (always evaluated for logging, cheaply)
hb_auc, hb_acc, hb_n = (_nan(), _nan(), 0)
hs_auc, hs_acc, hs_n = (_nan(), _nan(), 0)
if ho_base is not None:
hb_acc, hb_auc, hb_n = evaluate_baseline(baseline, ho_base, device, num_classes)
if ho_siam is not None:
hs_acc, hs_auc, hs_n = evaluate_siamese(siamese, ho_siam, device, num_classes)
# Best-epoch tracking: snapshot state independently per model
if not np.isnan(b_val_auc) and b_val_auc > best_base_auc:
best_base_auc = b_val_auc
best_base_val_acc = b_val_acc
best_base_state = copy.deepcopy(baseline.state_dict())
snap_ho_base_auc = hb_auc
snap_ho_base_acc = hb_acc
snap_ho_base_n = hb_n
if not np.isnan(s_val_auc) and s_val_auc > best_siam_auc:
best_siam_auc = s_val_auc
best_siam_val_acc = s_val_acc
best_siam_state = copy.deepcopy(siamese.state_dict())
snap_ho_siam_auc = hs_auc
snap_ho_siam_acc = hs_acc
snap_ho_siam_n = hs_n
best_epoch = epoch + 1
row = {
"fold": fold, "epoch": epoch + 1,
"base_train_loss": _f(bl_loss), "base_train_acc": _f(bl_acc),
"base_val_auc": _f(b_val_auc), "base_val_acc": _f(b_val_acc), "base_val_n": b_val_n,
"siam_train_loss": _f(si_loss), "siam_train_acc": _f(si_acc),
"siam_val_auc": _f(s_val_auc), "siam_val_acc": _f(s_val_acc), "siam_val_n": s_val_n,
"ho_base_auc": _f(hb_auc), "ho_base_acc": _f(hb_acc), "ho_base_n": hb_n,
"ho_siam_auc": _f(hs_auc), "ho_siam_acc": _f(hs_acc), "ho_siam_n": hs_n,
}
epoch_writer.writerow(row)
epoch_fp.flush()
if args.log_every > 0 and (epoch + 1) % args.log_every == 0:
print(
f" ep {epoch+1:>3}/{args.epochs} "
f"base val AUC={b_val_auc:.4f} siam val AUC={s_val_auc:.4f} "
f"(best base={best_base_auc:.4f} best siam={best_siam_auc:.4f})",
flush=True,
)
epoch_fp.close()
# Save best checkpoints
if best_base_state is not None:
torch.save(best_base_state, fold_dir / "best_baseline.pt")
if best_siam_state is not None:
torch.save(best_siam_state, fold_dir / "best_siamese.pt")
result = FoldResult(
mode=mode, fold=fold,
best_epoch=best_epoch,
baseline_best_val_auc=best_base_auc,
baseline_best_val_acc=best_base_val_acc,
siamese_best_val_auc=best_siam_auc,
siamese_best_val_acc=best_siam_val_acc,
baseline_holdout_auc=snap_ho_base_auc,
baseline_holdout_acc=snap_ho_base_acc,
baseline_holdout_n=snap_ho_base_n,
siamese_holdout_auc=snap_ho_siam_auc,
siamese_holdout_acc=snap_ho_siam_acc,
siamese_holdout_n=snap_ho_siam_n,
baseline_n=len(od_val),
siamese_n=len(bilat_val),
)
print(
f" [fold {fold+1}] BEST "
f"base val AUC={best_base_auc:.4f} acc={best_base_val_acc:.4f} "
f"siam val AUC={best_siam_auc:.4f} acc={best_siam_val_acc:.4f} "
f"(siam best epoch={best_epoch})",
flush=True,
)
if snap_ho_base_n > 0 or snap_ho_siam_n > 0:
print(
f" [fold {fold+1}] HOUT "
f"base AUC={snap_ho_base_auc:.4f} acc={snap_ho_base_acc:.4f} (n={snap_ho_base_n}) "
f"siam AUC={snap_ho_siam_auc:.4f} acc={snap_ho_siam_acc:.4f} (n={snap_ho_siam_n})",
flush=True,
)
return result
# ---------------------------------------------------------------------------
# Summary helpers
# ---------------------------------------------------------------------------
def _summary(results: list[FoldResult]) -> dict:
def _means(vals):
v = np.array([x for x in vals if not np.isnan(x)], dtype=float)
return (float(np.mean(v)) if len(v) else None,
float(np.std(v)) if len(v) else None)
b_val_aucs = [r.baseline_best_val_auc for r in results]
s_val_aucs = [r.siamese_best_val_auc for r in results]
b_ho_aucs = [r.baseline_holdout_auc for r in results]
s_ho_aucs = [r.siamese_holdout_auc for r in results]
b_val_accs = [r.baseline_best_val_acc for r in results]
s_val_accs = [r.siamese_best_val_acc for r in results]
b_ho_accs = [r.baseline_holdout_acc for r in results]
s_ho_accs = [r.siamese_holdout_acc for r in results]
deltas_val_auc = [s - b for b, s in zip(b_val_aucs, s_val_aucs)
if not np.isnan(b) and not np.isnan(s)]
deltas_ho_auc = [s - b for b, s in zip(b_ho_aucs, s_ho_aucs)
if not np.isnan(b) and not np.isnan(s)]
bva_m, bva_s = _means(b_val_aucs)
sva_m, sva_s = _means(s_val_aucs)
bha_m, bha_s = _means(b_ho_aucs)
sha_m, sha_s = _means(s_ho_aucs)
return {
"baseline_best_val": {"auc_mean": bva_m, "auc_std": bva_s, "acc_mean": _means(b_val_accs)[0]},
"siamese_best_val": {"auc_mean": sva_m, "auc_std": sva_s, "acc_mean": _means(s_val_accs)[0]},
"delta_val_auc": {"mean": float(np.mean(deltas_val_auc)) if deltas_val_auc else None,
"std": float(np.std(deltas_val_auc)) if deltas_val_auc else None},
"baseline_holdout": {"auc_mean": bha_m, "auc_std": bha_s, "acc_mean": _means(b_ho_accs)[0]},
"siamese_holdout": {"auc_mean": sha_m, "auc_std": sha_s, "acc_mean": _means(s_ho_accs)[0]},
"delta_holdout_auc": {"mean": float(np.mean(deltas_ho_auc)) if deltas_ho_auc else None,
"std": float(np.std(deltas_ho_auc)) if deltas_ho_auc else None},
}
def _print_summary(mode: str, s: dict) -> None:
def f(v):
return "nan" if v is None else f"{v:.4f}"
bv = s["baseline_best_val"]
sv = s["siamese_best_val"]
dv = s["delta_val_auc"]
bh = s["baseline_holdout"]
sh = s["siamese_holdout"]
dh = s["delta_holdout_auc"]
print(f"\n=== Summary [{mode}] (best-epoch metrics) ===")
print(f" val baseline AUC={f(bv['auc_mean'])}±{f(bv['auc_std'])} acc={f(bv['acc_mean'])}")
print(f" val siamese AUC={f(sv['auc_mean'])}±{f(sv['auc_std'])} acc={f(sv['acc_mean'])}")
print(f" val delta AUC={f(dv['mean'])}±{f(dv['std'])}")
print(f" hout baseline AUC={f(bh['auc_mean'])}±{f(bh['auc_std'])} acc={f(bh['acc_mean'])}")
print(f" hout siamese AUC={f(sh['auc_mean'])}±{f(sh['auc_std'])} acc={f(sh['acc_mean'])}")
print(f" hout delta AUC={f(dh['mean'])}±{f(dh['std'])}")
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def parse_args():
ap = argparse.ArgumentParser(
description="Baseline single-eye vs SiameseImageTower bilateral comparison."
)
ap.add_argument("--image-dir", default="Papila/FundusImages")
ap.add_argument("--clinical-dir", default="Papila/ClinicalData")
ap.add_argument("--label-col", default="Diagnosis")
ap.add_argument("--cat-cols", nargs="*", default=["Gender", "Phakic/Pseudophakic"])
ap.add_argument("--eval-mode", choices=["binary", "multiclass"], default="binary")
ap.add_argument(
"--eval-modes", nargs="+", choices=["binary", "multiclass"], default=None,
help="Run multiple modes in one pass, e.g. --eval-modes binary multiclass",
)
ap.add_argument("--n-splits", type=int, default=5)
ap.add_argument("--fold-seed", type=int, default=42)
ap.add_argument("--holdout-per-class", type=int, default=0)
ap.add_argument("--holdout-seed", type=int, default=123)
ap.add_argument("--folds", type=int, default=5)
ap.add_argument("--epochs", type=int, default=40)
ap.add_argument("--batch-size", type=int, default=8)
ap.add_argument("--lr", type=float, default=1e-4)
ap.add_argument("--backbone", default="refugelike")
ap.add_argument("--freeze-ratio", type=float, default=0.0)
ap.add_argument("--augment", action="store_true")
ap.add_argument("--num-workers", type=int, default=0)
ap.add_argument("--device", choices=["auto", "cpu", "cuda"], default="auto")
ap.add_argument("--seed", type=int, default=1234)
ap.add_argument("--run-name", default=None)
ap.add_argument("--output-root", default="analysis_data/basic_analysis")
ap.add_argument(
"--exclude-mixed-patients", action="store_true",
help="Drop patients whose two eyes have different labels before splitting.",
)
ap.add_argument(
"--log-every", type=int, default=5,
help="Print epoch progress every N epochs (0 to disable).",
)
return ap.parse_args()
def choose_device(name: str) -> torch.device:
if name == "cuda":
if not torch.cuda.is_available():
raise RuntimeError("--device cuda requested but CUDA is not available.")
return torch.device("cuda")
if name == "cpu":
return torch.device("cpu")
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------
def main():
args = parse_args()
device = choose_device(args.device)
seed_everything(args.seed)
print(f"Device: {device}", flush=True)
print("Loading PAPILA data...", flush=True)
data = build_papila_data(
image_dir=args.image_dir,
clinical_dir=args.clinical_dir,
label_col=args.label_col,
cat_cols=list(args.cat_cols),
n_splits=args.n_splits,
random_seed=args.fold_seed,
)
print(f"Loaded: {len(data.df)} rows", flush=True)
ts = time.strftime("%Y%m%d_%H%M%S")
run_name = args.run_name or f"siamese_compare_{ts}"
out_dir = Path(args.output_root) / run_name
out_dir.mkdir(parents=True, exist_ok=True)
eval_modes = args.eval_modes if args.eval_modes else [args.eval_mode]
all_results: dict[str, list[FoldResult]] = {}
summaries: dict[str, dict] = {}
for mode in eval_modes:
import pandas as pd
df_mode = data.df.copy()
if args.exclude_mixed_patients:
before = df_mode["Patient ID"].nunique()
df_mode, mixed = _drop_mixed_label_patients(
df_mode, patient_col="Patient ID", label_col=args.label_col
)
print(f"[{mode}] dropped {len(mixed)} mixed-label patients "
f"({before} -> {df_mode['Patient ID'].nunique()})", flush=True)
if mode == "binary":
df_mode = df_mode[df_mode[args.label_col].isin([0, 1])].reset_index(drop=True)
num_classes = 2 if mode == "binary" else int(df_mode[args.label_col].nunique())
print(
f"\n[{mode}] num_classes={num_classes} rows={len(df_mode)} "
f"patients={df_mode['Patient ID'].nunique()}",
flush=True,
)
split_manager = PatientFirstSplitManager(
patient_col="Patient ID", label_col=args.label_col
)
split_args = SimpleNamespace(
eval_mode=mode,
holdout_per_class=args.holdout_per_class,
holdout_seed=args.holdout_seed,
n_splits=args.n_splits,
fold_seed=args.fold_seed,
)
clinical_ns = SimpleNamespace(df=df_mode, label_col=args.label_col)
plans = split_manager.build_plans(clinical=clinical_ns, args=split_args, profile=None)
n_folds = min(args.folds, len(plans))
profile_od = build_papila_profile(
patient_col="Patient ID", label_col=args.label_col, sample_mode="eye"
)
profile_patient = build_papila_profile(
patient_col="Patient ID", label_col=args.label_col, sample_mode="patient"
)
mode_dir = out_dir / mode
mode_dir.mkdir(exist_ok=True)
fold_results: list[FoldResult] = []
for fold in range(n_folds):
fold_seed = args.seed + fold * 100
seed_everything(fold_seed)
fold_dir = mode_dir / f"fold{fold}"
fold_dir.mkdir(exist_ok=True)
print(f"\n[{mode}] fold {fold+1}/{n_folds}", flush=True)
result = run_fold(
fold=fold,
split=plans[fold],
mode=mode,
args=args,
device=device,
data=data,
num_classes=num_classes,
profile_od=profile_od,
profile_patient=profile_patient,
fold_dir=fold_dir,
)
fold_results.append(result)
# Write per-mode CSV
fold_csv = out_dir / f"{mode}_fold_results.csv"
csv_fields = list(FoldResult.__dataclass_fields__.keys())
with fold_csv.open("w", newline="", encoding="utf-8") as fh:
w = csv.DictWriter(fh, fieldnames=csv_fields)
w.writeheader()
for r in fold_results:
w.writerow({k: getattr(r, k) for k in csv_fields})
summary = _summary(fold_results)
_print_summary(mode, summary)
all_results[mode] = fold_results
summaries[mode] = summary
payload = {
"run_name": run_name,
"timestamp": ts,
"config": vars(args),
"summaries": summaries,
}
(out_dir / "summary.json").write_text(json.dumps(payload, indent=2), encoding="utf-8")
print(f"\nOutputs written to: {out_dir}")
if __name__ == "__main__":
main()
scripts/basic_analysis/compare_siamese_v2.py
+1025
View File
File diff suppressed because it is too large Load Diff
scripts/basic_analysis/papila_univariate_roc.py
Executable
+264
View File
@@ -0,0 +1,264 @@
#!/usr/bin/env python3
"""Univariate ROC curves for PAPILA clinical variables."""
import re
from pathlib import Path
from typing import Iterable, List, Tuple
import sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, auc
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes import build_papila_clinical
# ---------------------------
# Config (edit in IDE)
# ---------------------------
IMAGE_DIR = "Papila/FundusImages"
CLINICAL_DIR = "Papila/ClinicalData"
LABEL_COL = "Diagnosis"
CAT_COLS = ["Gender", "Phakic/Pseudophakic"]
EXCLUDE_COLS = {"Pneumatic", "Perkins"}
INCLUDE_CATEGORICAL = False
POSITIVE_LABEL = 1
NEGATIVE_LABEL = 0
DROP_LABELS = [2]
OUTPUT_DIR = Path("analysis_data/basic_analysis/papila_univariate_roc")
DEBUG_PRINTS = False
PLOT_PER_FEATURE = False
DI_OPTRE_COL_PREFIXES = ("dioptre",)
ADD_DIOPTRE_ABS = True
ADD_DIOPTRE_SQUARED = True
def _sanitize(name: str) -> str:
safe = re.sub(r"[^A-Za-z0-9._-]+", "_", str(name)).strip("_")
return safe or "var"
def _select_binary_labels(labels: pd.Series,
positive_label: int,
negative_label: int,
drop_labels: Iterable[int]) -> Tuple[np.ndarray, np.ndarray]:
labels_num = pd.to_numeric(labels, errors="coerce")
use_num = labels_num.notna().any()
lab = labels_num if use_num else labels.astype(str)
drop_set = set(drop_labels or [])
keep = lab.isin([positive_label, negative_label])
if drop_set:
keep &= ~lab.isin(drop_set)
y = (lab == positive_label).astype(int)
return y.values, keep.values
def _compute_roc(y: np.ndarray, scores: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray, float]:
fpr, tpr, thresholds = roc_curve(y, scores, pos_label=1)
auc_val = float(auc(fpr, tpr))
return fpr, tpr, thresholds, auc_val
def _best_threshold(fpr: np.ndarray, tpr: np.ndarray, thresholds: np.ndarray) -> Tuple[float, float, float]:
youden = tpr - fpr
idx = int(np.nanargmax(youden))
return float(thresholds[idx]), float(tpr[idx]), float(fpr[idx])
def _plot_roc(fpr: np.ndarray, tpr: np.ndarray, auc_val: float, title: str, out_path: Path) -> None:
fig, ax = plt.subplots(figsize=(5.5, 4.5))
ax.plot(fpr, tpr, lw=1.8, label=f"AUC={auc_val:.3f}")
ax.plot([0, 1], [0, 1], "k--", lw=1)
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(title)
ax.legend(loc="lower right")
ax.grid(True, alpha=0.3, linestyle="--")
fig.tight_layout()
fig.savefig(out_path, dpi=170)
plt.close(fig)
def _plot_overlay(curves, title: str, out_path: Path) -> None:
fig, ax = plt.subplots(figsize=(7, 5.5))
cmap = plt.get_cmap("tab20")
for i, (name, fpr, tpr, auc_val) in enumerate(curves):
color = cmap(i % cmap.N)
ax.plot(fpr, tpr, lw=1.6, color=color, label=f"{name} (AUC={auc_val:.3f})")
ax.plot([0, 1], [0, 1], "k--", lw=1)
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(title)
ax.legend(loc="upper left", fontsize="small")
ax.grid(True, alpha=0.3, linestyle="--")
fig.tight_layout()
fig.savefig(out_path, dpi=170)
plt.close(fig)
def _iter_numeric(df: pd.DataFrame, cols: List[str]):
for col in cols:
if col not in df.columns:
continue
s = pd.to_numeric(df[col], errors="coerce")
yield col, s
def _is_dioptre_col(col: str) -> bool:
name = str(col).strip().lower()
return any(name.startswith(prefix) for prefix in DI_OPTRE_COL_PREFIXES)
def _iter_numeric_with_transforms(df: pd.DataFrame, cols: List[str]):
for col, s in _iter_numeric(df, cols):
yield col, s
if _is_dioptre_col(col):
if ADD_DIOPTRE_ABS:
yield f"{col}_abs", s.abs()
if ADD_DIOPTRE_SQUARED:
yield f"{col}_sq", s.pow(2)
def _include_in_overlay(feature_name: str) -> bool:
name = str(feature_name).strip().lower()
if _is_dioptre_col(name) and not name.endswith("_abs"):
return False
return True
def _iter_categorical(df: pd.DataFrame, cols: List[str]):
for col in cols:
if col not in df.columns:
continue
s = df[col]
vals = s.dropna().unique().tolist()
try:
vals = sorted(vals)
except Exception:
pass
for v in vals:
name = f"{col}=={v}"
ind = (s == v).astype(int)
yield name, ind
def main() -> None:
clinical = build_papila_clinical(
image_dir=IMAGE_DIR,
clinical_dir=CLINICAL_DIR,
label_col=LABEL_COL,
cat_cols=CAT_COLS,
)
df = clinical.df.copy()
labels = df[LABEL_COL]
y_all, keep_mask = _select_binary_labels(labels, POSITIVE_LABEL, NEGATIVE_LABEL, DROP_LABELS)
out_dir = OUTPUT_DIR
plot_dir = out_dir / "plots"
plot_dir.mkdir(parents=True, exist_ok=True)
rows = []
curves = []
base_exclude = {LABEL_COL, "Patient ID"} | EXCLUDE_COLS | set(CAT_COLS)
if "eyeID" not in CAT_COLS:
base_exclude.add("eyeID")
candidate_cols = [c for c in df.columns if c not in base_exclude]
numeric_cols = []
for col in candidate_cols:
s = pd.to_numeric(df[col], errors="coerce")
if s.notna().any():
numeric_cols.append(col)
if DEBUG_PRINTS:
for col in ("IOP_raw", "IOP_corr"):
if col not in df.columns:
print(f"[debug] {col} missing from df")
continue
s = pd.to_numeric(df[col], errors="coerce")
print(f"[debug] {col}: non-null={int(s.notna().sum())}, unique={int(s.nunique(dropna=True))}")
for col, series in _iter_numeric_with_transforms(df, numeric_cols):
mask = keep_mask & series.notna().values
y = y_all[mask]
scores = series.values[mask].astype(float)
if y.size < 2 or np.unique(y).size < 2:
continue
if np.nanmin(scores) == np.nanmax(scores):
continue
fpr, tpr, thresholds, auc_val = _compute_roc(y, scores)
thr, best_tpr, best_fpr = _best_threshold(fpr, tpr, thresholds)
direction = "high" if auc_val >= 0.5 else "low"
title = f"{col} (n={y.size}, direction={direction})"
if PLOT_PER_FEATURE:
out_path = plot_dir / f"roc_{_sanitize(col)}.png"
_plot_roc(fpr, tpr, auc_val, title, out_path)
if _include_in_overlay(col):
curves.append((col, fpr, tpr, auc_val))
rows.append({
"feature": col,
"kind": "numeric",
"n": int(y.size),
"auc": auc_val,
"direction": direction,
"best_threshold": thr,
"best_tpr": best_tpr,
"best_fpr": best_fpr,
"best_specificity": 1.0 - best_fpr,
})
if INCLUDE_CATEGORICAL:
cat_cols_use = [c for c in clinical.cat_cols if c not in EXCLUDE_COLS]
for name, ind in _iter_categorical(df, cat_cols_use):
mask = keep_mask & ind.notna().values
y = y_all[mask]
scores = ind.values[mask].astype(float)
if y.size < 2 or np.unique(y).size < 2:
continue
if np.nanmin(scores) == np.nanmax(scores):
continue
fpr, tpr, thresholds, auc_val = _compute_roc(y, scores)
thr, best_tpr, best_fpr = _best_threshold(fpr, tpr, thresholds)
direction = "high" if auc_val >= 0.5 else "low"
title = f"{name} (n={y.size}, direction={direction})"
if PLOT_PER_FEATURE:
out_path = plot_dir / f"roc_{_sanitize(name)}.png"
_plot_roc(fpr, tpr, auc_val, title, out_path)
if _include_in_overlay(name):
curves.append((name, fpr, tpr, auc_val))
rows.append({
"feature": name,
"kind": "categorical",
"n": int(y.size),
"auc": auc_val,
"direction": direction,
"best_threshold": thr,
"best_tpr": best_tpr,
"best_fpr": best_fpr,
"best_specificity": 1.0 - best_fpr,
})
if not rows:
raise SystemExit("No valid features produced ROC curves. Check labels and feature columns.")
overlay_path = plot_dir / "roc_overlay.png"
_plot_overlay(curves, "Univariate ROC curves", overlay_path)
out_df = pd.DataFrame(rows).sort_values(by="auc", ascending=False)
out_dir.mkdir(parents=True, exist_ok=True)
out_df.to_csv(out_dir / "summary.csv", index=False)
print(out_df.to_string(index=False, float_format=lambda x: f"{x:.4f}"))
print(f"\nSaved overlay plot to: {overlay_path}")
if PLOT_PER_FEATURE:
print(f"Saved per-feature plots to: {plot_dir}")
print(f"Saved summary to: {out_dir / 'summary.csv'}")
if __name__ == "__main__":
main()
scripts/deprecated/inspect_gt_masks.py
Executable
+142
View File
@@ -0,0 +1,142 @@
"""Generate ground-truth mask overlays for REFUGE and Papila samples."""
from __future__ import annotations
import argparse
from pathlib import Path
import sys
ROOT = Path(__file__).resolve().parents[1]
sys.path.append(str(ROOT))
import numpy as np
from collections import Counter
from PIL import Image
from PIL.Image import Resampling
from classes.unet_segmenter import UNetSegmenter
REFUGE_ROOT = Path("REFUGE")
DEFAULT_MANIFEST = Path("manifest.csv")
OUTPUT_DIR = Path("temp/gt_test")
def to_mask_colors(disc: np.ndarray, cup: np.ndarray) -> Image.Image:
h, w = disc.shape
canvas = np.ones((h, w, 3), dtype=np.uint8) * 255
disc_mask = disc.astype(bool)
cup_mask = cup.astype(bool)
canvas[disc_mask] = [128, 128, 128]
canvas[cup_mask] = [0, 0, 0]
return Image.fromarray(canvas)
def overlay(
original: Image.Image, mask_rgb: Image.Image, alpha: float = 0.6
) -> Image.Image:
mask_rgba = mask_rgb.convert("RGBA")
updates = np.array(mask_rgba, dtype=np.float32)
updates[..., 3] = alpha * 255 * (updates[..., :3] != 255).any(axis=-1)
base = original.convert("RGBA")
return Image.alpha_composite(
base, Image.fromarray(updates.astype(np.uint8))
).convert("RGB")
def original_mask_to_rgb(mask_path: Path) -> Image.Image:
mask_img = Image.open(mask_path)
arr = np.asarray(mask_img)
h, w = arr.shape[:2]
canvas = np.ones((h, w, 3), dtype=np.uint8) * 255
if arr.ndim == 2:
border = np.concatenate([arr[0, :], arr[-1, :], arr[:, 0], arr[:, -1]])
bg_value = Counter(border.tolist()).most_common(1)[0][0]
disc_mask = arr != bg_value
fg_counts = Counter(arr[arr != bg_value].flatten())
if fg_counts:
# For REFUGE-style masks: cup should be the darkest (minimum value)
cup_value = min(fg_counts.keys())
cup_mask = arr == cup_value
else:
cup_mask = np.zeros_like(arr, dtype=bool)
else:
edges = np.concatenate(
[arr[0, :, :], arr[-1, :, :], arr[:, 0, :], arr[:, -1, :]], axis=0
)
bg_color = Counter(map(tuple, edges)).most_common(1)[0][0]
disc_mask = ~np.all(arr == bg_color, axis=-1)
color_counts = Counter(map(tuple, arr.reshape(-1, arr.shape[2])))
cup_mask = np.zeros((h, w), dtype=bool)
candidates = {}
for color, count in color_counts.items():
if color == bg_color:
continue
mask = np.all(arr == color, axis=-1)
candidates[color] = mask
if candidates:
# For REFUGE-style masks: cup should be the darkest color (closest to black)
cup_color = min(candidates.keys(), key=lambda color: sum(color))
cup_mask = candidates[cup_color]
disc_mask = disc_mask.astype(bool)
cup_mask = cup_mask & disc_mask
canvas[disc_mask] = [128, 128, 128]
canvas[cup_mask] = [0, 0, 0]
return Image.fromarray(canvas)
def process_entries(
segmenter: UNetSegmenter, entries, prefix: str, count: int, dest: Path
) -> None:
for entry in entries[:count]:
img_path = Path(entry.image_path)
if not img_path.exists():
continue
orig = Image.open(img_path).convert("RGB")
image = segmenter.preprocess_image(orig)
disc, cup = segmenter.load_masks(entry)
disc_coords = segmenter._mask_to_coords(disc)
cup_coords = segmenter._mask_to_coords(cup)
print(
f"{entry.sample_id}: disc coords {disc_coords.shape[0] if disc_coords is not None else 0}, "
f"cup coords {cup_coords.shape[0] if cup_coords is not None else 0}"
)
mask_rgb = to_mask_colors(disc, cup)
overlay_img = overlay(image, mask_rgb)
mask_rgb.save(dest / f"{prefix}_{entry.sample_id}_mask.png")
overlay_img.save(dest / f"{prefix}_{entry.sample_id}_overlay.png")
if entry.annotation_type_disc == "mask":
gt_mask_rgb = original_mask_to_rgb(entry.annotation_disc)
gt_overlay = overlay(
orig.resize(gt_mask_rgb.size, Resampling.BILINEAR), gt_mask_rgb
)
gt_mask_rgb.save(dest / f"{prefix}_{entry.sample_id}_gt_mask.png")
gt_overlay.save(dest / f"{prefix}_{entry.sample_id}_gt_overlay.png")
def main() -> None:
parser = argparse.ArgumentParser(
description="Inspect ground-truth masks for REFUGE and Papila"
)
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
parser.add_argument("--output", type=Path, default=OUTPUT_DIR)
parser.add_argument(
"--count", type=int, default=10, help="Number of samples per dataset"
)
args = parser.parse_args()
args.output.mkdir(parents=True, exist_ok=True)
segmenter = UNetSegmenter(args.manifest)
refuge_entries = [e for e in segmenter._manifest if e.dataset == "refuge"]
papila_entries = [e for e in segmenter._manifest if e.dataset == "papila"]
process_entries(segmenter, refuge_entries, "refuge", args.count, args.output)
process_entries(segmenter, papila_entries, "papila", args.count, args.output)
print(f"Saved overlays to {args.output}")
if __name__ == "__main__":
main()
scripts/grid_search_analytics/best_holdout_multiclass.py
Executable
+117
View File
@@ -0,0 +1,117 @@
#!/usr/bin/env python3
"""Rank multiclass runs by mean holdout AUC (fused) across folds."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional
import numpy as np
import pandas as pd
# ---------------------------
# Config (edit in IDE)
# ---------------------------
ANALYSIS_DIR = Path("analysis_data/grid_search")
TOP_N = 20
HEAD = "fused" # fused | image | metadata
OUTPUT_CSV = Path("analysis_data/grid_search/plots/best_holdout_multiclass.csv")
def _read_json(path: Path) -> Optional[Dict[str, Any]]:
if not path.exists():
return None
try:
data = json.loads(path.read_text())
except Exception:
return None
return data if isinstance(data, dict) else None
def _infer_mode(summary: Optional[Dict[str, Any]]) -> Optional[str]:
if not summary:
return None
eval_mode = summary.get("eval_mode")
if isinstance(eval_mode, str):
mode = eval_mode.strip().lower()
if mode == "binary":
return "binary"
if mode in {"multiclass", "multi", "multi-class"}:
return "multiclass"
num_classes = summary.get("num_classes")
if isinstance(num_classes, (int, float)):
return "binary" if int(num_classes) <= 2 else "multiclass"
class_names = summary.get("class_names")
if isinstance(class_names, list) and class_names:
return "binary" if len(class_names) <= 2 else "multiclass"
return None
def _simple_fields(summary: Dict[str, Any]) -> Dict[str, Any]:
keep: Dict[str, Any] = {}
for key, val in summary.items():
if key == "fold_metrics":
continue
if isinstance(val, (str, int, float, bool)) or val is None:
keep[key] = val
return keep
def _collect_fold_values(summary: Dict[str, Any], metric_key: str) -> List[float]:
values: List[float] = []
for entry in summary.get("fold_metrics") or []:
if not isinstance(entry, dict):
continue
stats = entry.get("stats") if isinstance(entry.get("stats"), dict) else {}
val = stats.get(metric_key)
if isinstance(val, (int, float)):
values.append(float(val))
return values
def main() -> None:
metric_key = f"holdout_auc_{HEAD}"
rows: List[Dict[str, Any]] = []
for run_dir in sorted(ANALYSIS_DIR.iterdir()):
if not run_dir.is_dir():
continue
summary = _read_json(run_dir / "summary.json")
mode = _infer_mode(summary)
if mode != "multiclass":
continue
values = _collect_fold_values(summary, metric_key)
if not values:
continue
mean_val = float(np.mean(values))
std_val = float(np.std(values, ddof=1)) if len(values) > 1 else float("nan")
row = {
"run_id": summary.get("run_id", run_dir.name),
"run_dir": str(run_dir),
"metric": metric_key,
"mean": mean_val,
"std": std_val,
"n_folds": len(values),
**_simple_fields(summary),
}
rows.append(row)
if not rows:
raise SystemExit("No multiclass runs with holdout AUC found.")
df = pd.DataFrame(rows).sort_values(by="mean", ascending=False)
top_df = df.head(TOP_N) if TOP_N else df
OUTPUT_CSV.parent.mkdir(parents=True, exist_ok=True)
df.to_csv(OUTPUT_CSV, index=False)
print(top_df.to_string(index=False, float_format=lambda x: f"{x:.4f}"))
print(f"\nSaved full ranking to: {OUTPUT_CSV}")
if __name__ == "__main__":
main()
scripts/grid_search_analytics/check_crop_cache_vs_gt.py
Executable
+376
View File
@@ -0,0 +1,376 @@
#!/usr/bin/env python3
"""Compare cached crop bounds/features vs GT-derived crops from the manifest."""
from __future__ import annotations
from pathlib import Path
from typing import Dict, List, Optional, Tuple
import sys
import numpy as np
import pandas as pd
from PIL import Image
import torch
from torchvision import transforms
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.hypertower import ManifestImageCropper, UNetImageCropper
from classes.refuge_segmentation import UNet as RefugeUNet
# ---------------------------
# Config (edit in IDE)
# ---------------------------
CACHE_DIR = Path("analysis_data/hypertower_crops")
MANIFEST_PATH = Path("manifest.csv")
IMAGE_DIR = Path("Papila/FundusImages")
SCALE = 2.5
MAX_SAMPLES = 200 # set None to scan all
TOL_BOUNDS = 1.0 # pixels
TOL_FEATURES = 1e-3
UNET_VARIANTS = [
("norm_imagenet", Path("models/unet_segmenter/norm_imagenet/best.pt"), "imagenet"),
("normalize_none", Path("models/unet_segmenter/normalize_none/best.pt"), "none"),
("norm_per_image", Path("models/unet_segmenter/norm_per_image/best.pt"), "per_image"),
]
REFUGE_SEG_WEIGHTS = Path("models/refuge/segmentation/refuge_segmentation_best.pt")
def _load_cache(path: Path) -> Optional[Dict[str, np.ndarray]]:
try:
data = np.load(path, allow_pickle=False)
except Exception:
return None
return {k: data[k] for k in data.files}
def _parse_stem(path: Path) -> str:
# expects RET###OS_s250.npz -> RET###OS
stem = path.stem
if "_s" in stem:
stem = stem.split("_s")[0]
return stem
def _image_path_from_stem(stem: str) -> Optional[Path]:
cand = IMAGE_DIR / f"{stem}.jpg"
if cand.exists():
return cand
cand = IMAGE_DIR / f"{stem}.png"
if cand.exists():
return cand
return None
def _gt_info(
cropper: ManifestImageCropper, image_path: Path
) -> Optional[Dict[str, float]]:
try:
image = Image.open(image_path).convert("RGB")
except Exception:
return None
info = cropper._compute_crop_info(image, image_path)
return info
def _unet_info(
cropper: UNetImageCropper, image_path: Path
) -> Optional[Dict[str, float]]:
try:
image = Image.open(image_path).convert("RGB")
except Exception:
return None
info = cropper._compute_crop_info(image, image_path)
return info
def _load_refuge_model(device: str) -> Optional[RefugeUNet]:
if not REFUGE_SEG_WEIGHTS.exists():
return None
model = RefugeUNet()
try:
state = torch.load(REFUGE_SEG_WEIGHTS, map_location=device)
except Exception:
return None
state_dict = state.get("model", state) if isinstance(state, dict) else state
try:
model.load_state_dict(state_dict)
except Exception:
return None
model.to(device)
model.eval()
return model
def _refuge_seg_info(
model: RefugeUNet, device: str, image_path: Path
) -> Optional[Dict[str, float]]:
try:
image = Image.open(image_path).convert("RGB")
except Exception:
return None
original_size = image.size
image_resized = image.resize((512, 512), Image.BILINEAR)
tensor = transforms.ToTensor()(image_resized).unsqueeze(0).to(device)
with torch.no_grad():
logits = model(tensor)
mask = torch.sigmoid(logits)[0, 0]
mask_np = (mask.cpu().numpy() > 0.5).astype(np.float32)
mask_img = Image.fromarray(mask_np)
mask_img = mask_img.resize(original_size, Image.NEAREST)
mask_np = np.array(mask_img, dtype=np.float32)
coords = np.argwhere(mask_np > 0.5)
if coords.size == 0:
return None
ys, xs = coords[:, 0], coords[:, 1]
centre_x = float(xs.mean())
centre_y = float(ys.mean())
width = float(xs.max() - xs.min())
height = float(ys.max() - ys.min())
diameter = max(width, height)
radius = diameter / 2.0
crop_radius = radius * SCALE
left = max(0.0, centre_x - crop_radius)
upper = max(0.0, centre_y - crop_radius)
right = min(float(image.width), centre_x + crop_radius)
lower = min(float(image.height), centre_y + crop_radius)
return {
"left": left,
"upper": upper,
"right": right,
"lower": lower,
}
def _diff_bounds(cache: Dict[str, np.ndarray], gt: Dict[str, float]) -> Optional[float]:
keys = ("left", "upper", "right", "lower")
if not all(k in cache for k in keys):
return None
diffs = [abs(float(cache[k]) - float(gt[k])) for k in keys]
return float(max(diffs))
def _diff_features(
cache: Dict[str, np.ndarray], gt: Dict[str, float]
) -> Optional[float]:
if "features" not in cache or "features" not in gt:
return None
cf = np.asarray(cache["features"], dtype=float).ravel()
gf = np.asarray(gt["features"], dtype=float).ravel()
if cf.shape != gf.shape:
return None
return float(np.max(np.abs(cf - gf)))
def main() -> None:
if not CACHE_DIR.exists():
raise SystemExit(f"Cache dir not found: {CACHE_DIR}")
if not MANIFEST_PATH.exists():
raise SystemExit(f"Manifest not found: {MANIFEST_PATH}")
cache_files = sorted(CACHE_DIR.glob(f"*_s{int(SCALE * 100)}.npz"))
if MAX_SAMPLES is not None:
cache_files = cache_files[:MAX_SAMPLES]
print(f"[debug] cache files found: {len(cache_files)}")
try:
manifest_df = pd.read_csv(MANIFEST_PATH)
except Exception as exc:
raise SystemExit(f"Failed to read manifest: {exc}")
manifest_images = manifest_df.get("image_path")
if manifest_images is None:
raise SystemExit("Manifest is missing image_path column.")
manifest_images = manifest_images.dropna().astype(str)
manifest_stems = {Path(p).stem for p in manifest_images}
print(f"[debug] manifest image_path count: {len(manifest_images)}")
print(f"[debug] manifest unique stems: {len(manifest_stems)}")
cache_stems = {_parse_stem(p) for p in cache_files}
overlap = cache_stems & manifest_stems
print(
f"[debug] cache stems: {len(cache_stems)} overlap with manifest stems: {len(overlap)}"
)
if cache_files:
print(f"[debug] example cache stems: {sorted(list(cache_stems))[:5]}")
if manifest_stems:
print(f"[debug] example manifest stems: {sorted(list(manifest_stems))[:5]}")
cropper = ManifestImageCropper(
manifest_path=MANIFEST_PATH,
scale=SCALE,
target_size=224,
cache_dir=None,
)
rows: List[Dict[str, object]] = []
for cache_path in cache_files:
cache = _load_cache(cache_path)
if cache is None:
continue
stem = _parse_stem(cache_path)
image_path = _image_path_from_stem(stem)
if image_path is None:
continue
gt = _gt_info(cropper, image_path)
if gt is None:
continue
bounds_diff = _diff_bounds(cache, gt)
feat_diff = _diff_features(cache, gt)
rows.append(
{
"file": cache_path.name,
"bounds_diff": bounds_diff,
"features_diff": feat_diff,
"bounds_match": bounds_diff is not None and bounds_diff <= TOL_BOUNDS,
"features_match": feat_diff is not None and feat_diff <= TOL_FEATURES,
}
)
if not rows:
print("[warn] No cache entries matched GT manifest entries.")
else:
df = pd.DataFrame(rows)
print(df.head(10).to_string(index=False))
print("\nSummary:")
print(df[["bounds_diff", "features_diff"]].describe().to_string())
if df["bounds_match"].notna().any():
match_rate = df["bounds_match"].mean()
print(f"\nBounds match rate (<= {TOL_BOUNDS}px): {match_rate:.3f}")
if df["features_match"].notna().any():
match_rate = df["features_match"].mean()
print(f"Features match rate (<= {TOL_FEATURES}): {match_rate:.3f}")
print("\nUNet variant comparisons (no cache writes):")
for name, weights, normalize in UNET_VARIANTS:
if not weights.exists():
print(f"[warn] {name}: weights not found at {weights}")
continue
unet = UNetImageCropper(
manifest_path=MANIFEST_PATH,
weights_path=weights,
normalize=normalize,
threshold=0.5,
tta=False,
scale=SCALE,
target_size=224,
cache_dir=None, # ensure no cache writes
)
u_rows: List[Dict[str, object]] = []
missing_images = 0
unet_none = 0
cache_missing = 0
exceptions = 0
for cache_path in cache_files:
cache = _load_cache(cache_path)
if cache is None:
cache_missing += 1
continue
stem = _parse_stem(cache_path)
image_path = _image_path_from_stem(stem)
if image_path is None:
missing_images += 1
continue
try:
info = _unet_info(unet, image_path)
except Exception:
exceptions += 1
continue
if info is None:
unet_none += 1
continue
bounds_diff = _diff_bounds(cache, info)
feat_diff = _diff_features(cache, info)
u_rows.append(
{
"bounds_diff": bounds_diff,
"features_diff": feat_diff,
"bounds_match": bounds_diff is not None
and bounds_diff <= TOL_BOUNDS,
"features_match": feat_diff is not None
and feat_diff <= TOL_FEATURES,
}
)
if not u_rows:
print(
f"[warn] {name}: no comparisons computed "
f"(cache_missing={cache_missing}, missing_images={missing_images}, "
f"unet_none={unet_none}, exceptions={exceptions})"
)
continue
u_df = pd.DataFrame(u_rows)
b_mean = float(u_df["bounds_diff"].mean())
f_mean = float(u_df["features_diff"].mean())
b_match = float(u_df["bounds_match"].mean())
f_match = float(u_df["features_match"].mean())
print(
f"{name}: mean bounds diff={b_mean:.3f}, mean feat diff={f_mean:.6f}, "
f"bounds match rate={b_match:.3f}, features match rate={f_match:.3f}"
)
print("\nRefuge segmentation model comparison (bounds only, no cache writes):")
device = "cuda" if torch.cuda.is_available() else "cpu"
refuge_model = _load_refuge_model(device)
if refuge_model is None:
print(f"[warn] refuge_segmentation_best.pt not found or failed to load at {REFUGE_SEG_WEIGHTS}")
return
r_rows: List[Dict[str, object]] = []
missing_images = 0
cache_missing = 0
model_none = 0
exceptions = 0
for cache_path in cache_files:
cache = _load_cache(cache_path)
if cache is None:
cache_missing += 1
continue
stem = _parse_stem(cache_path)
image_path = _image_path_from_stem(stem)
if image_path is None:
missing_images += 1
continue
try:
info = _refuge_seg_info(refuge_model, device, image_path)
except Exception:
exceptions += 1
continue
if info is None:
model_none += 1
continue
bounds_diff = _diff_bounds(cache, info)
r_rows.append(
{
"bounds_diff": bounds_diff,
"bounds_match": bounds_diff is not None and bounds_diff <= TOL_BOUNDS,
}
)
if not r_rows:
print(
"[warn] refuge_segmentation_best: no comparisons computed "
f"(cache_missing={cache_missing}, missing_images={missing_images}, "
f"model_none={model_none}, exceptions={exceptions})"
)
return
r_df = pd.DataFrame(r_rows)
b_mean = float(r_df["bounds_diff"].mean())
b_match = float(r_df["bounds_match"].mean())
print(
f"refuge_segmentation_best: mean bounds diff={b_mean:.3f}, "
f"bounds match rate={b_match:.3f}"
)
if __name__ == "__main__":
main()
scripts/grid_search_analytics/compare_suspect_geometry_vs_image.py
Executable
+249
View File
@@ -0,0 +1,249 @@
#!/usr/bin/env python3
"""Compare suspect AUC from image tower vs crop-derived geometry (CDR)."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Dict, List, Optional, Tuple
import sys
import numpy as np
import pandas as pd
from PIL import Image
from sklearn.metrics import roc_auc_score
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes import build_papila_clinical
from classes.hypertower import UNetImageCropper, ManifestImageCropper
# ---------------------------
# Config (edit in IDE)
# ---------------------------
RUN_DIRS = [
Path("analysis_data/1030_Balanced_Unet_Perimg_Resnet_SE16NormB_SE16NormT_multi_fused/1030_Balanced_Unet_Perimg_Resnet_SE16NormB_SE16NormT_multi_fused_20251030_091842"),
Path("analysis_data/1030_Balanced_GT_Perimg_Resnet_SE16NormB_SE16NormT_multi_fused/1030_Balanced_GT_Perimg_Resnet_SE16NormB_SE16NormT_multi_fused_20251030_113730"),
]
GEOM_CACHE_ROOT = Path("analysis_data/geometry_cache")
SUSPECT_LABEL = 2
def _load_json(path: Path) -> Dict:
if not path.exists():
return {}
try:
return json.loads(path.read_text())
except Exception:
return {}
def _drop_holdout_rows(clinical, holdout_path: Path) -> None:
if not holdout_path.exists():
return
holdout = pd.read_csv(holdout_path)
if holdout.empty:
return
if "Patient ID" not in holdout.columns or "eyeID" not in holdout.columns:
return
holdout_keys = set(zip(holdout["Patient ID"].astype(int), holdout["eyeID"].astype(str)))
df = clinical.df.copy()
df["_key"] = list(zip(df["Patient ID"].astype(int), df["eyeID"].astype(str)))
df = df[~df["_key"].isin(holdout_keys)].drop(columns=["_key"]).reset_index(drop=True)
clinical.frames = [df.copy()]
clinical.df = df.copy()
clinical._infer_or_validate_feature_types()
clinical._compute_numeric_stats()
clinical._build_cat_maps()
clinical._compute_feature_dim()
clinical._build_kfold_indices()
def _make_cropper(args: Dict, cache_dir: Path):
manifest = args.get("img_crop_manifest")
if not manifest:
raise RuntimeError("img_crop_manifest missing; cannot compute geometry features.")
scale = float(args.get("img_crop_scale", 2.5))
target_size = int(args.get("img_crop_size", 224))
use_gt = bool(args.get("img_crop_gt", False))
if use_gt:
return ManifestImageCropper(
manifest_path=Path(manifest),
scale=scale,
target_size=target_size,
cache_dir=cache_dir,
)
weights = args.get("img_crop_weights")
if not weights:
raise RuntimeError("img_crop_weights missing for UNet cropper.")
normalize = args.get("img_crop_normalize", "per_image")
threshold = float(args.get("img_crop_threshold", 0.5))
tta = bool(args.get("img_crop_tta", False))
return UNetImageCropper(
manifest_path=Path(manifest),
weights_path=Path(weights),
normalize=normalize,
threshold=threshold,
tta=tta,
scale=scale,
target_size=target_size,
cache_dir=cache_dir,
)
def _geometry_scores(
clinical,
cropper,
test_df: pd.DataFrame,
) -> Tuple[np.ndarray, np.ndarray]:
scores: List[float] = []
keep_mask: List[bool] = []
for _, row in test_df.iterrows():
img_path = clinical.get_image_path(row)
try:
image = Image.open(img_path).convert("RGB")
except Exception:
scores.append(float("nan"))
keep_mask.append(False)
continue
feats = cropper.geometry_features(image, img_path)
if feats is None or len(feats) == 0:
scores.append(float("nan"))
keep_mask.append(False)
else:
scores.append(float(feats[0])) # area_ratio (CDR)
keep_mask.append(True)
return np.asarray(scores, dtype=float), np.asarray(keep_mask, dtype=bool)
def _suspect_auc(y_true: np.ndarray, scores: np.ndarray) -> float:
y = (y_true == SUSPECT_LABEL).astype(int)
if y.sum() == 0 or y.sum() == len(y):
return float("nan")
return float(roc_auc_score(y, scores))
def main() -> None:
rows: List[Dict[str, object]] = []
for run_dir in RUN_DIRS:
cli_path = run_dir / "cli_args.json"
cli_args = _load_json(cli_path)
if not cli_args:
print(f"[warn] Missing cli_args.json in {run_dir}")
continue
label_col = cli_args.get("label_col", "Diagnosis")
cat_cols = cli_args.get("cat_cols", ["Gender", "Phakic/Pseudophakic"])
n_splits = int(cli_args.get("n_splits", 5))
fold_seed = int(cli_args.get("fold_seed", 42))
eval_mode = str(cli_args.get("eval_mode", "multiclass")).lower()
clinical = build_papila_clinical(
image_dir=cli_args.get("image_dir", "Papila/FundusImages"),
clinical_dir=cli_args.get("clinical_dir", "Papila/ClinicalData"),
label_col=label_col,
cat_cols=cat_cols,
n_splits=n_splits,
random_seed=fold_seed,
)
if eval_mode == "binary":
clinical.df = clinical.df[clinical.df[label_col].isin([0, 1])].reset_index(drop=True)
clinical.frames = [clinical.df.copy()]
clinical._infer_or_validate_feature_types()
clinical._compute_numeric_stats()
clinical._build_cat_maps()
clinical._compute_feature_dim()
clinical._build_kfold_indices()
_drop_holdout_rows(clinical, run_dir / "holdout.csv")
cache_dir = GEOM_CACHE_ROOT / run_dir.name
cache_dir.mkdir(parents=True, exist_ok=True)
cropper = _make_cropper(cli_args, cache_dir=cache_dir)
all_geom_scores: List[float] = []
all_img_scores: List[float] = []
all_y: List[int] = []
for fold in range(n_splits):
y_path = run_dir / f"fold{fold}_y_true.npy"
p_img_path = run_dir / f"fold{fold}_probs_img.npy"
if not y_path.exists() or not p_img_path.exists():
continue
y_true = np.load(y_path)
probs_img = np.load(p_img_path)
if probs_img.ndim != 2 or probs_img.shape[1] <= SUSPECT_LABEL:
continue
_, test_df = clinical.get_split_dfs(fold)
if len(test_df) != len(y_true):
print(
f"[warn] {run_dir.name} fold{fold}: test_df len {len(test_df)} != y_true len {len(y_true)}"
)
geom_scores, keep_mask = _geometry_scores(clinical, cropper, test_df)
if keep_mask.sum() == 0:
print(f"[warn] {run_dir.name} fold{fold}: no valid geometry features")
continue
y_fold = y_true[: len(geom_scores)][keep_mask]
geom_fold = geom_scores[keep_mask]
img_fold = probs_img[: len(geom_scores), SUSPECT_LABEL][keep_mask]
geom_auc = _suspect_auc(y_fold, geom_fold)
img_auc = _suspect_auc(y_fold, img_fold)
rows.append(
{
"run": run_dir.name,
"fold": fold,
"metric": "suspect_auc",
"image_auc": img_auc,
"geometry_auc": geom_auc,
"n": int(len(y_fold)),
}
)
all_geom_scores.append(geom_fold)
all_img_scores.append(img_fold)
all_y.append(y_fold)
if all_y:
y_all = np.concatenate(all_y)
geom_all = np.concatenate(all_geom_scores)
img_all = np.concatenate(all_img_scores)
rows.append(
{
"run": run_dir.name,
"fold": "all",
"metric": "suspect_auc",
"image_auc": _suspect_auc(y_all, img_all),
"geometry_auc": _suspect_auc(y_all, geom_all),
"n": int(len(y_all)),
}
)
if not rows:
raise SystemExit("No results produced; check run paths and files.")
df = pd.DataFrame(rows)
out_path = GEOM_CACHE_ROOT / "suspect_auc_geometry_vs_image.csv"
out_path.parent.mkdir(parents=True, exist_ok=True)
df.to_csv(out_path, index=False)
print(df.to_string(index=False, float_format=lambda x: f"{x:.4f}"))
print(f"\nSaved: {out_path}")
if __name__ == "__main__":
main()
scripts/grid_search_analytics/derived_analysis.py
Executable
+1826
View File
File diff suppressed because it is too large Load Diff
scripts/grid_search_analytics/derived_statistics.py
Executable
+314
View File
@@ -0,0 +1,314 @@
#!/usr/bin/env python3
"""
Shared helpers for building grid search analytics.
The class below will gradually accumulate reusable utilities for working with
grid search outputs (summary.json, cli_args.json, etc).
"""
from __future__ import annotations
import json
import csv
import re
from pathlib import Path
from typing import Dict, Iterable, Iterator, List, Optional
import numpy as np
DEFAULT_EXCLUDE_KEYS = {
"run_id",
"fold_metrics",
"best_metric",
"best_metric_mode",
"best_metric_mean",
"best_metric_std",
"eval_mode",
"n_splits",
"num_classes",
}
class GridSearchAnalytics:
"""Utility wrapper for inspecting grid search result directories."""
def __init__(self,
analysis_dir: Path | str,
exclude_keys: Optional[Iterable[str]] = None) -> None:
self.analysis_dir = Path(analysis_dir)
if not self.analysis_dir.exists():
raise FileNotFoundError(f"analysis_dir does not exist: {self.analysis_dir}")
self.exclude_keys = set(exclude_keys or DEFAULT_EXCLUDE_KEYS)
def iter_run_dirs(self, shallow: bool = True) -> Iterator[Path]:
"""
Yield run directories containing grid search artifacts.
Shallow iteration only walks direct children. Deep iteration scans the
entire subtree.
"""
candidates: Iterable[Path]
if shallow:
candidates = (p for p in sorted(self.analysis_dir.iterdir()) if p.is_dir())
else:
candidates = (p for p in self.analysis_dir.rglob("*") if p.is_dir())
for run_dir in candidates:
summary = run_dir / "summary.json"
cli = run_dir / "cli_args.json"
if summary.exists() or cli.exists():
yield run_dir
def read_summary(self, run_dir: Path) -> Optional[Dict[str, object]]:
"""Load summary.json for a run directory."""
return self._read_json(run_dir / "summary.json")
def read_cli_args(self, run_dir: Path) -> Optional[Dict[str, object]]:
"""Load cli_args.json for a run directory."""
return self._read_json(run_dir / "cli_args.json")
def read_run_id(self,
run_dir: Path,
summary: Optional[Dict[str, object]]) -> str:
if summary:
rid = summary.get("run_id")
if isinstance(rid, str) and rid:
return rid
return run_dir.name
def task_from_summary(self, summary: Optional[Dict[str, object]]) -> Optional[str]:
"""Infer task (binary vs multiclass) from a summary payload."""
if not summary:
return None
eval_mode = summary.get("eval_mode")
if isinstance(eval_mode, str):
mode = eval_mode.strip().lower()
if mode == "binary":
return "binary"
if mode in {"multiclass", "multi", "multi-class"}:
return "multiclass"
num_classes = summary.get("num_classes")
if isinstance(num_classes, (int, float)):
return "binary" if int(num_classes) <= 2 else "multiclass"
return None
def flatten_config(self,
data: Dict[str, object],
prefix: str = "",
exclude_keys: Optional[Iterable[str]] = None) -> Dict[str, object]:
"""Flatten nested CLI args or config dictionaries for analysis."""
out: Dict[str, object] = {}
excludes = set(exclude_keys or self.exclude_keys)
for key, value in data.items():
if key in excludes or key.startswith("best_"):
continue
full_key = f"{prefix}{key}" if not prefix else f"{prefix}.{key}"
if isinstance(value, dict):
out.update(self.flatten_config(value, full_key, exclude_keys=excludes))
continue
if isinstance(value, list):
continue
out[full_key] = value
return out
def fusion_correction_events(self,
output_csv: Path | str | None = None,
shallow: bool = True) -> Path:
"""
Build a table of cases where the fused head is correct while both towers
are wrong. Rows are written to CSV for downstream analysis.
"""
output_path = Path(output_csv) if output_csv else Path("analysis_data/grid_search_analytics/fusion_corrections.csv")
output_path.parent.mkdir(parents=True, exist_ok=True)
rows: List[Dict[str, object]] = []
for run_dir in self.iter_run_dirs(shallow=shallow):
summary = self.read_summary(run_dir)
run_id = self.read_run_id(run_dir, summary)
folds = self._available_folds(run_dir, summary)
for fold in folds:
y_true = self._load_y_true(run_dir, fold)
if y_true is None:
continue
epoch_prob_paths = self._collect_epoch_prob_paths(run_dir, fold)
if not epoch_prob_paths:
# Per-epoch dumps were not found; fall back to the saved fold-level probabilities.
base_paths = self._collect_base_prob_paths(run_dir, fold)
if base_paths:
epoch_hint = self._fold_epoch_hint(summary, fold)
epoch_prob_paths = {epoch_hint if epoch_hint is not None else 0: base_paths}
for epoch, paths in epoch_prob_paths.items():
arrays = {head: self._load_probs_array(path) for head, path in paths.items()}
if not self._has_all_heads(arrays):
continue
events = self._fusion_corrections_for_probs(y_true, arrays, run_id, fold, epoch)
rows.extend(events)
if rows:
fieldnames = [
"run_id",
"fold",
"epoch",
"index",
"y_true",
"pred_fused",
"pred_img",
"pred_md",
"conf_fused",
"conf_img",
"conf_md",
]
with output_path.open("w", newline="") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
else:
output_path.write_text("")
return output_path
@staticmethod
def _read_json(path: Path) -> Optional[Dict[str, object]]:
if not path.exists():
return None
try:
data = json.loads(path.read_text())
except Exception:
return None
if not isinstance(data, dict):
return None
return data
@staticmethod
def _fold_epoch_hint(summary: Optional[Dict[str, object]], fold: int) -> Optional[int]:
if not summary:
return None
fold_metrics = summary.get("fold_metrics") or []
for entry in fold_metrics:
if not isinstance(entry, dict):
continue
if entry.get("fold") == fold:
stats = entry.get("stats") if isinstance(entry.get("stats"), dict) else {}
epoch = stats.get("epoch") or entry.get("best_epoch")
if isinstance(epoch, (int, float)):
return int(epoch)
return None
@staticmethod
def _available_folds(run_dir: Path, summary: Optional[Dict[str, object]]) -> List[int]:
folds: List[int] = []
if summary:
for entry in summary.get("fold_metrics") or []:
if not isinstance(entry, dict):
continue
fold_idx = entry.get("fold")
if isinstance(fold_idx, int):
folds.append(fold_idx)
if not folds:
pattern = re.compile(r"fold(\d+)_y_true\.npy$")
for path in run_dir.glob("fold*_y_true.npy"):
match = pattern.match(path.name)
if match:
folds.append(int(match.group(1)))
return sorted(set(folds))
@staticmethod
def _load_y_true(run_dir: Path, fold: int) -> Optional[np.ndarray]:
path = run_dir / f"fold{fold}_y_true.npy"
if not path.exists():
return None
try:
return np.load(path)
except Exception:
return None
@staticmethod
def _collect_epoch_prob_paths(run_dir: Path, fold: int) -> Dict[int, Dict[str, Path]]:
pattern = re.compile(rf"fold{fold}_epoch(\d+)_probs_(\w+)\.npy$")
epoch_paths: Dict[int, Dict[str, Path]] = {}
for path in run_dir.glob(f"fold{fold}_epoch*_probs_*.npy"):
match = pattern.match(path.name)
if not match:
continue
epoch = int(match.group(1))
head = match.group(2)
epoch_paths.setdefault(epoch, {})[head] = path
return epoch_paths
@staticmethod
def _collect_base_prob_paths(run_dir: Path, fold: int) -> Dict[str, Path]:
paths: Dict[str, Path] = {}
for head in ("fused", "img", "md"):
candidate = run_dir / f"fold{fold}_probs_{head}.npy"
if candidate.exists():
paths[head] = candidate
return paths
@staticmethod
def _load_probs_array(path: Path) -> Optional[np.ndarray]:
try:
return np.load(path)
except Exception:
return None
@staticmethod
def _prepare_probs(arr: np.ndarray) -> Optional[np.ndarray]:
if arr is None:
return None
probs = np.asarray(arr, dtype=float)
if probs.ndim == 1:
probs = np.stack([1.0 - probs, probs], axis=1)
if probs.ndim != 2:
return None
return probs
@staticmethod
def _has_all_heads(arrays: Dict[str, Optional[np.ndarray]]) -> bool:
needed = ("fused", "img", "md")
return all(arrays.get(head) is not None for head in needed)
def _fusion_corrections_for_probs(self,
y_true: np.ndarray,
arrays: Dict[str, np.ndarray],
run_id: str,
fold: int,
epoch: int) -> List[Dict[str, object]]:
fused = self._prepare_probs(arrays.get("fused"))
img = self._prepare_probs(arrays.get("img"))
md = self._prepare_probs(arrays.get("md"))
if fused is None or img is None or md is None:
return []
if not (len(fused) == len(img) == len(md) == len(y_true)):
return []
fused_pred = fused.argmax(axis=1)
img_pred = img.argmax(axis=1)
md_pred = md.argmax(axis=1)
fused_conf = np.take_along_axis(fused, fused_pred[:, None], axis=1).squeeze(1)
img_conf = np.take_along_axis(img, img_pred[:, None], axis=1).squeeze(1)
md_conf = np.take_along_axis(md, md_pred[:, None], axis=1).squeeze(1)
mask = (fused_pred == y_true) & (img_pred != y_true) & (md_pred != y_true)
indices = np.nonzero(mask)[0]
events: List[Dict[str, object]] = []
for idx in indices:
events.append({
"run_id": run_id,
"fold": fold,
"epoch": epoch,
"index": int(idx),
"y_true": int(y_true[idx]),
"pred_fused": int(fused_pred[idx]),
"pred_img": int(img_pred[idx]),
"pred_md": int(md_pred[idx]),
"conf_fused": float(fused_conf[idx]),
"conf_img": float(img_conf[idx]),
"conf_md": float(md_conf[idx]),
})
return events
if __name__ == "__main__":
analytics = GridSearchAnalytics(Path("analysis_data/grid_search"))
output = analytics.fusion_correction_events()
print(f"Fusion correction events written to {output}")
scripts/grid_search_analytics/grid_search_heatmap.py
Executable
+729
View File
@@ -0,0 +1,729 @@
#!/usr/bin/env python3
"""
Build an HTML heatmap-style grid for grid search runs.
Each column is a run. The header shows mean metrics (auc, acc, holdout_auc,
holdout_acc). Rows encode hyperparameter options as red/green boxes.
Example:
python scripts/grid_search_analytics/grid_search_heatmap.py \
--analysis-dir analysis_data/grid_search \
--task binary \
--sort-by holdout_auc --desc \
--top 40 \
--format plot \
--output analysis_data/grid_search_heatmap.png
"""
from __future__ import annotations
import argparse
import json
import math
import os
import sys
import time
from html import escape
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Tuple
DEFAULT_EXCLUDE_KEYS = {
"run_id",
"fold_metrics",
"best_metric",
"best_metric_mode",
"best_metric_mean",
"best_metric_std",
"eval_mode",
"n_splits",
"num_classes",
}
def to_float(value: Optional[object]) -> Optional[float]:
if value is None:
return None
if isinstance(value, (int, float)):
num = float(value)
if math.isnan(num):
return None
return num
if not isinstance(value, str):
return None
value = value.strip()
if not value:
return None
try:
num = float(value)
except ValueError:
return None
if math.isnan(num):
return None
return num
def mean(values: List[float]) -> Optional[float]:
return (sum(values) / len(values)) if values else None
def read_summary(run_dir: Path) -> Optional[Dict[str, object]]:
summary_path = run_dir / "summary.json"
if not summary_path.exists():
return None
try:
data = json.loads(summary_path.read_text())
except Exception:
return None
if not isinstance(data, dict):
return None
return data
def read_cli_args(run_dir: Path) -> Optional[Dict[str, object]]:
cli_path = run_dir / "cli_args.json"
if not cli_path.exists():
return None
try:
data = json.loads(cli_path.read_text())
except Exception:
return None
if not isinstance(data, dict):
return None
return data
def read_run_id(run_dir: Path, summary: Optional[Dict[str, object]]) -> str:
if summary:
rid = summary.get("run_id")
if isinstance(rid, str) and rid:
return rid
return run_dir.name
def task_from_summary(summary: Optional[Dict[str, object]]) -> Optional[str]:
if not summary:
return None
eval_mode = summary.get("eval_mode")
if isinstance(eval_mode, str):
mode = eval_mode.strip().lower()
if mode == "binary":
return "binary"
if mode in {"multiclass", "multi", "multi-class"}:
return "multiclass"
num_classes = summary.get("num_classes")
if isinstance(num_classes, (int, float)):
return "binary" if int(num_classes) <= 2 else "multiclass"
return None
def metric_from_stats(stats: Dict[str, object], metric: str) -> Optional[float]:
if metric.startswith("holdout_") and stats.get("holdout_best_monitor") == metric:
best_val = to_float(stats.get("holdout_best_so_far"))
if best_val is not None:
return best_val
return to_float(stats.get(metric))
def mean_metric(summary: Dict[str, object], metric: str) -> Optional[float]:
folds = summary.get("fold_metrics") or []
if not isinstance(folds, list) or not folds:
return None
values = []
for fold in folds:
stats = fold.get("stats") if isinstance(fold, dict) else None
if not isinstance(stats, dict):
return None
val = metric_from_stats(stats, metric)
if val is None:
return None
values.append(val)
return mean(values)
def flatten_config(data: Dict[str, object],
prefix: str = "",
exclude_keys: Optional[Iterable[str]] = None) -> Dict[str, object]:
out: Dict[str, object] = {}
excludes = set(exclude_keys or [])
for key, value in data.items():
if key in excludes or key.startswith("best_"):
continue
full_key = f"{prefix}{key}" if not prefix else f"{prefix}.{key}"
if isinstance(value, dict):
out.update(flatten_config(value, full_key, exclude_keys=excludes))
continue
if isinstance(value, list):
continue
out[full_key] = value
return out
def sort_value_key(value: object) -> Tuple[int, object]:
if value is None:
return (2, "")
if isinstance(value, bool):
return (0, int(value))
if isinstance(value, (int, float)):
return (0, value)
return (1, str(value))
def format_value(value: object) -> str:
if value is None:
return ""
if isinstance(value, bool):
return "true" if value else "false"
if isinstance(value, int):
return str(value)
if isinstance(value, float):
return f"{value:.6g}"
return str(value)
def format_metric(value: Optional[float]) -> str:
if value is None:
return ""
return f"{value:.4f}"
def render_progress(current: int, total: Optional[int], matched: int) -> str:
if total:
width = 30
filled = int(width * current / total)
bar = "#" * filled + "-" * (width - filled)
return f"[{bar}] {current}/{total} matched {matched}"
return f"Scanned {current} dirs, matched {matched}"
def iter_run_dirs(root: Path, shallow: bool, show_progress: bool) -> Iterable[Path]:
if shallow:
entries = [entry for entry in root.iterdir() if entry.is_dir()]
entries.sort(key=lambda p: p.name)
total = len(entries)
matched = 0
last_update = 0.0
for idx, entry in enumerate(entries, start=1):
if show_progress:
now = time.monotonic()
if now - last_update >= 0.1 or idx == total:
msg = render_progress(idx, total, matched)
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
last_update = now
if (entry / "summary.json").is_file():
matched += 1
yield entry
if show_progress:
print(file=sys.stderr)
return
matched = 0
scanned = 0
last_update = 0.0
for dirpath, dirnames, filenames in os.walk(root):
scanned += 1
if show_progress:
now = time.monotonic()
if now - last_update >= 0.2:
msg = render_progress(scanned, None, matched)
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
last_update = now
if "summary.json" in filenames:
matched += 1
yield Path(dirpath)
if show_progress:
msg = render_progress(scanned, None, matched)
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
print(file=sys.stderr)
def build_html(runs: List[Dict[str, object]],
row_specs: List[Tuple[str, object]],
title: str,
filters: List[str]) -> str:
lines: List[str] = []
lines.append("<!doctype html>")
lines.append("<html lang=\"en\">")
lines.append("<head>")
lines.append("<meta charset=\"utf-8\">")
lines.append(f"<title>{escape(title)}</title>")
lines.append("<style>")
lines.append(":root { --green: #4caf50; --red: #d9534f; --grid: #d0d0d0; --header: #f0f0f0; }")
lines.append("body { margin: 0; padding: 16px; font-family: \"Courier New\", monospace; }")
lines.append(".wrap { overflow-x: auto; }")
lines.append("table { border-collapse: collapse; font-size: 12px; }")
lines.append("th, td { border: 1px solid var(--grid); padding: 4px; text-align: center; }")
lines.append("th.row-label { text-align: left; background: var(--header); position: sticky; left: 0; }")
lines.append("thead th { background: var(--header); position: sticky; top: 0; z-index: 1; }")
lines.append("th.run-id { writing-mode: vertical-rl; transform: rotate(180deg); white-space: nowrap; }")
lines.append("td.cell { width: 14px; height: 14px; padding: 0; }")
lines.append("td.on { background: var(--green); }")
lines.append("td.off { background: var(--red); }")
lines.append(".meta { margin-bottom: 12px; }")
lines.append("</style>")
lines.append("</head>")
lines.append("<body>")
lines.append(f"<h2>{escape(title)}</h2>")
if filters:
lines.append("<div class=\"meta\">")
for item in filters:
lines.append(f"<div>{escape(item)}</div>")
lines.append("</div>")
lines.append("<div class=\"wrap\">")
lines.append("<table>")
lines.append("<thead>")
lines.append("<tr>")
lines.append("<th>run_id</th>")
for run in runs:
run_id = escape(str(run.get("run_id", "")))
rel_path = escape(str(run.get("relative_path", "")))
title_attr = f" title=\"{rel_path}\"" if rel_path else ""
lines.append(f"<th class=\"run-id\"{title_attr}>{run_id}</th>")
lines.append("</tr>")
for metric_key, label in [
("auc", "auc"),
("acc", "acc"),
("holdout_auc", "holdout_auc"),
("holdout_acc", "holdout_acc"),
]:
lines.append("<tr>")
lines.append(f"<th>{label}</th>")
for run in runs:
metrics = run.get("metrics", {})
value = metrics.get(metric_key) if isinstance(metrics, dict) else None
lines.append(f"<td>{escape(format_metric(value))}</td>")
lines.append("</tr>")
lines.append("</thead>")
lines.append("<tbody>")
for key, value in row_specs:
label = f"{key}={format_value(value)}"
lines.append("<tr>")
lines.append(f"<th class=\"row-label\">{escape(label)}</th>")
for run in runs:
config = run.get("config", {})
current = config.get(key) if isinstance(config, dict) else None
cell_class = "on" if current == value else "off"
lines.append(f"<td class=\"cell {cell_class}\"></td>")
lines.append("</tr>")
lines.append("</tbody>")
lines.append("</table>")
lines.append("</div>")
lines.append("</body>")
lines.append("</html>")
return "\n".join(lines)
def truncate(text: str, width: int) -> str:
if len(text) <= width:
return text
if width <= 3:
return text[:width]
return text[:width - 3] + "..."
def build_text_grid(runs: List[Dict[str, object]],
row_specs: List[Tuple[str, object]],
filters: List[str],
col_width: int,
row_width: int,
color: bool) -> str:
sep = " "
lines: List[str] = []
if filters:
lines.extend(filters)
lines.append("")
def pad(text: str, width: int) -> str:
return truncate(text, width).ljust(width)
def colorize(text: str, enabled: bool) -> str:
if not color:
return text
color_code = "\x1b[32m" if enabled else "\x1b[31m"
return f"{color_code}{text}\x1b[0m"
def row_line(label: str, values: List[str]) -> str:
return pad(label, row_width) + sep + sep.join(pad(v, col_width) for v in values)
run_ids = [str(run.get("run_id", "")) for run in runs]
lines.append(row_line("run_id", run_ids))
for metric_key, label in [
("auc", "auc"),
("acc", "acc"),
("holdout_auc", "holdout_auc"),
("holdout_acc", "holdout_acc"),
]:
values = []
for run in runs:
metrics = run.get("metrics", {})
value = metrics.get(metric_key) if isinstance(metrics, dict) else None
values.append(format_metric(value))
lines.append(row_line(label, values))
divider = "-" * row_width + sep + sep.join("-" * col_width for _ in runs)
lines.append(divider)
for key, value in row_specs:
label = f"{key}={format_value(value)}"
cells: List[str] = []
for run in runs:
config = run.get("config", {})
current = config.get(key) if isinstance(config, dict) else None
enabled = current == value
cell = colorize("##", enabled) if enabled else colorize("..", enabled)
cells.append(cell)
lines.append(row_line(label, cells))
lines.append("")
lines.append("Legend: ##=on ..=off")
if color:
lines.append("Colors: green=on red=off")
return "\n".join(lines)
def parse_figsize(value: Optional[str], n_cols: int, n_rows: int) -> Tuple[float, float]:
if value:
parts = [p.strip() for p in value.split(",") if p.strip()]
if len(parts) == 2:
try:
return float(parts[0]), float(parts[1])
except ValueError:
pass
width = min(40.0, max(8.0, n_cols * 0.3))
height = min(40.0, max(6.0, (n_rows + 6) * 0.3))
return width, height
def plot_heatmap(runs: List[Dict[str, object]],
row_specs: List[Tuple[str, object]],
filters: List[str],
output_path: Optional[Path],
figsize: Tuple[float, float],
dpi: int,
show: bool) -> None:
if not show:
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
try:
import seaborn as sns
except ImportError:
sns = None
metric_labels = ["auc", "acc", "holdout_auc", "holdout_acc"]
metric_matrix: List[List[float]] = []
for label in metric_labels:
row: List[float] = []
for run in runs:
metrics = run.get("metrics", {})
value = metrics.get(label) if isinstance(metrics, dict) else None
row.append(float(value) if value is not None else float("nan"))
metric_matrix.append(row)
param_labels = [f"{key}={format_value(value)}" for key, value in row_specs]
param_matrix: List[List[int]] = []
for key, value in row_specs:
row = []
for run in runs:
config = run.get("config", {})
current = config.get(key) if isinstance(config, dict) else None
row.append(1 if current == value else 0)
param_matrix.append(row)
fig = plt.figure(figsize=figsize, dpi=dpi)
grid_rows = 2 if param_matrix else 1
height_ratios = [2, max(2, len(param_matrix) * 0.5)] if param_matrix else [2]
gs = fig.add_gridspec(grid_rows, 1, height_ratios=height_ratios, hspace=0.05)
ax_metrics = fig.add_subplot(gs[0, 0])
if sns:
sns.heatmap(
metric_matrix,
ax=ax_metrics,
cmap="viridis",
annot=True,
fmt=".3f",
cbar=True,
yticklabels=metric_labels,
xticklabels=False,
)
else:
im = ax_metrics.imshow(metric_matrix, aspect="auto", cmap="viridis")
ax_metrics.set_yticks(range(len(metric_labels)))
ax_metrics.set_yticklabels(metric_labels)
fig.colorbar(im, ax=ax_metrics, fraction=0.02, pad=0.01)
for i, row in enumerate(metric_matrix):
for j, value in enumerate(row):
if math.isnan(value):
continue
ax_metrics.text(j, i, f"{value:.3f}", ha="center", va="center", fontsize=7, color="white")
ax_metrics.set_ylabel("metrics")
if param_matrix:
ax_params = fig.add_subplot(gs[1, 0], sharex=ax_metrics)
cmap = ListedColormap(["#d9534f", "#4caf50"])
if sns:
sns.heatmap(
param_matrix,
ax=ax_params,
cmap=cmap,
cbar=False,
yticklabels=param_labels,
xticklabels=[run.get("run_id", "") for run in runs],
vmin=0,
vmax=1,
)
else:
ax_params.imshow(param_matrix, aspect="auto", cmap=cmap, vmin=0, vmax=1)
ax_params.set_yticks(range(len(param_labels)))
ax_params.set_yticklabels(param_labels)
ax_params.set_xticks(range(len(runs)))
ax_params.set_xticklabels([run.get("run_id", "") for run in runs], rotation=90)
ax_params.set_xlabel("runs")
else:
ax_metrics.set_xticks(range(len(runs)))
ax_metrics.set_xticklabels([run.get("run_id", "") for run in runs], rotation=90)
ax_metrics.set_xlabel("runs")
if filters:
fig.suptitle("Grid Search Heatmap\n" + " | ".join(filters), fontsize=10)
else:
fig.suptitle("Grid Search Heatmap", fontsize=10)
if output_path is not None:
output_path.parent.mkdir(parents=True, exist_ok=True)
fig.savefig(output_path, bbox_inches="tight")
if show:
plt.show()
plt.close(fig)
def main() -> None:
ap = argparse.ArgumentParser(description="Build an HTML heatmap grid for grid search runs.")
ap.add_argument("--analysis-dir", type=Path, default=Path("analysis_data/grid_search"),
help="Directory containing run subdirectories")
ap.add_argument("--format", choices=["text", "html", "plot"], default="text",
help="Output format (default: text)")
ap.add_argument("--task", choices=["binary", "multiclass", "all"], default="all",
help="Filter runs by task type (default: all)")
ap.add_argument("--sort-by", choices=["auc", "acc", "holdout_auc", "holdout_acc"], default=None,
help="Metric to sort columns by (default: none)")
ap.add_argument("--asc", action="store_true",
help="Sort in ascending order (default: descending)")
ap.add_argument("--desc", action="store_true",
help="Sort in descending order (default: descending)")
ap.add_argument("--top", type=int, default=None,
help="Limit to the top N runs after sorting")
ap.add_argument("--cluster-rows", dest="cluster_rows", action="store_true",
help="Order parameter rows by prevalence in the selected runs (default)")
ap.add_argument("--no-cluster-rows", dest="cluster_rows", action="store_false",
help="Keep parameter rows sorted alphabetically")
ap.set_defaults(cluster_rows=True)
ap.add_argument("--output", type=Path, default=None,
help="Optional path to write output")
ap.add_argument("--params", default=None,
help="Comma-separated list of parameter keys to include")
ap.add_argument("--exclude", default=None,
help="Comma-separated list of parameter keys to exclude")
ap.add_argument("--match", default=None,
help="Only include run directories whose name contains this substring")
ap.add_argument("--shallow", action="store_true",
help="Only scan directories directly under analysis-dir")
ap.add_argument("--no-progress", action="store_true",
help="Disable progress output")
ap.add_argument("--col-width", type=int, default=13,
help="Column width for text output (default: 13)")
ap.add_argument("--row-width", type=int, default=36,
help="Row label width for text output (default: 36)")
ap.add_argument("--color", action="store_true",
help="Use ANSI colors in text output")
ap.add_argument("--figsize", default=None,
help="Figure size as 'width,height' (inches), for plot output")
ap.add_argument("--dpi", type=int, default=140,
help="Figure DPI for plot output")
ap.add_argument("--show", action="store_true",
help="Display plot window (only for format=plot)")
args = ap.parse_args()
root = args.analysis_dir
if not root.exists():
raise SystemExit(f"Analysis directory not found: {root}")
exclude_keys = set(DEFAULT_EXCLUDE_KEYS)
if args.exclude:
for item in args.exclude.split(","):
item = item.strip()
if item:
exclude_keys.add(item)
runs: List[Dict[str, object]] = []
values_by_key: Dict[str, List[object]] = {}
missing_summary = 0
unknown_task = 0
missing_cli = 0
for run_dir in iter_run_dirs(root, shallow=args.shallow, show_progress=not args.no_progress):
if args.match and args.match not in run_dir.name:
continue
summary = read_summary(run_dir)
if summary is None:
missing_summary += 1
continue
task_label = task_from_summary(summary)
if args.task != "all":
if task_label is None:
unknown_task += 1
continue
if task_label != args.task:
continue
metrics = {
"auc": mean_metric(summary, "auc_fused"),
"acc": mean_metric(summary, "acc_fused"),
"holdout_auc": mean_metric(summary, "holdout_auc_fused"),
"holdout_acc": mean_metric(summary, "holdout_acc_fused"),
}
if any(val is None for val in metrics.values()):
continue
cli_args = read_cli_args(run_dir)
if cli_args is None:
missing_cli += 1
config_source = cli_args if cli_args is not None else summary
config = flatten_config(config_source, exclude_keys=exclude_keys)
run_id = read_run_id(run_dir, summary)
runs.append({
"run_id": run_id,
"relative_path": str(run_dir.relative_to(root)),
"task": task_label,
"metrics": metrics,
"config": config,
})
for key, value in config.items():
values_by_key.setdefault(key, []).append(value)
if not runs:
print("No matching runs found.")
return
if args.params:
param_keys = [p.strip() for p in args.params.split(",") if p.strip()]
else:
param_keys = []
for key, values in values_by_key.items():
unique_values = {format_value(v) for v in values}
if len(unique_values) > 1:
param_keys.append(key)
param_keys.sort()
row_specs: List[Tuple[str, object]] = []
for key in param_keys:
values = values_by_key.get(key, [])
unique_values = []
seen = set()
for val in values:
marker = (type(val), val)
if marker in seen:
continue
seen.add(marker)
unique_values.append(val)
unique_values.sort(key=sort_value_key)
for value in unique_values:
row_specs.append((key, value))
if args.asc and args.desc:
raise SystemExit("Choose only one of --asc or --desc.")
if args.sort_by:
def sort_key(item: Dict[str, object]) -> float:
metrics = item.get("metrics", {})
val = metrics.get(args.sort_by) if isinstance(metrics, dict) else None
if val is None:
return float("inf") if args.asc else float("-inf")
return float(val)
runs.sort(key=sort_key, reverse=not args.asc)
else:
runs.sort(key=lambda r: str(r.get("run_id", "")))
if args.top is not None:
runs = runs[:args.top]
if args.cluster_rows and runs:
total = len(runs)
counts_by_spec: Dict[Tuple[str, object], int] = {}
for key, value in row_specs:
counts_by_spec[(key, value)] = 0
for run in runs:
config = run.get("config", {})
if not isinstance(config, dict):
continue
for key, value in row_specs:
if config.get(key) == value:
counts_by_spec[(key, value)] += 1
def row_sort(spec: Tuple[str, object]) -> Tuple[float, str, str]:
count = counts_by_spec.get(spec, 0)
score = count / total if total else 0.0
key, value = spec
return (-score, str(key), format_value(value))
row_specs.sort(key=row_sort)
filters = []
if args.task != "all":
filters.append(f"Task filter: {args.task}")
if args.match:
filters.append(f"Name filter: {args.match}")
filters.append(f"Runs: {len(runs)}")
filters.append(f"Params: {len(row_specs)}")
filters.append(f"Row clustering: {'on' if args.cluster_rows else 'off'}")
if missing_summary or unknown_task:
filters.append(f"Skipped: {missing_summary} missing summary, {unknown_task} unknown task")
if missing_cli:
filters.append(f"Missing cli_args: {missing_cli}")
title = "Grid Search Heatmap"
if args.format == "html":
html = build_html(runs, row_specs, title=title, filters=filters)
output_path = args.output or Path("analysis_data/grid_search_heatmap.html")
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(html)
print(f"Wrote {output_path}")
elif args.format == "plot":
output_path = args.output
if output_path is None and not args.show:
output_path = Path("analysis_data/grid_search_heatmap.png")
figsize = parse_figsize(args.figsize, n_cols=len(runs), n_rows=len(row_specs))
plot_heatmap(
runs,
row_specs,
filters=filters,
output_path=output_path,
figsize=figsize,
dpi=args.dpi,
show=args.show,
)
if output_path is not None:
print(f"Wrote {output_path}")
else:
text = build_text_grid(
runs,
row_specs,
filters=filters,
col_width=max(4, args.col_width),
row_width=max(12, args.row_width),
color=args.color,
)
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(text)
print(f"Wrote {args.output}")
else:
print(text)
if __name__ == "__main__":
main()
scripts/grid_search_analytics/holdout_roc_for_run.py
Executable
+572
View File
@@ -0,0 +1,572 @@
#!/usr/bin/env python3
"""Plot holdout ROC curves for a specific grid search run."""
from __future__ import annotations
import json
import re
from pathlib import Path
from typing import Dict, List, Tuple
import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics import auc, roc_curve
# ---------------------------
# Config (edit in IDE)
# ---------------------------
RUN_ID = "20251129-0312"
ANALYSIS_ROOT = Path("analysis_data/grid_search")
HEADS = ["fused", "image", "metadata"]
OUTPUT_SUBDIR = Path("plots/holdout_rocs")
BEST_OUTPUT_SUBDIR = Path("plots/best_rocs")
POSITIVE_CLASS = 1
DEBUG = True
USE_JSON_ROC = True
USE_HOLDOUT_PROBS = True
ALLOW_FALLBACK_TO_VALIDATION = False
PLOT_VALIDATION_FROM_HOLDOUT_EPOCH = True
PLOT_BEST_EPOCH = True
PLOT_HOLDOUT_FROM_BEST_EPOCH = True
PLOT_ALL_CLASSES = True
FORCE_PROBS_FOR_BEST_BINARY = True
FORCE_PROBS_FOR_HOLDOUT_BINARY = False
HEAD_FILE_KEYS = {
"fused": "fused",
"image": "img",
"metadata": "md",
}
JSON_DIR_NAMES = [
"roc_curves_holdout_best",
"roc_curves",
]
def _epoch_from_name(path: Path) -> int:
m = re.search(r"epoch(\d+)", path.name)
return int(m.group(1)) if m else -1
def _load_json(path: Path) -> Dict:
try:
return json.loads(path.read_text())
except Exception:
return {}
def _infer_run_info(run_dir: Path) -> Tuple[str | None, int | None, List[str] | None]:
cli = _load_json(run_dir / "cli_args.json")
summary = _load_json(run_dir / "summary.json")
payloads = [cli, summary]
eval_mode = None
num_classes = None
class_names = None
for payload in payloads:
if not payload:
continue
if eval_mode is None:
em = payload.get("eval_mode")
if isinstance(em, str):
eval_mode = em.strip().lower()
if num_classes is None:
nc = payload.get("num_classes")
if isinstance(nc, (int, float)):
num_classes = int(nc)
if class_names is None:
cn = payload.get("class_names")
if isinstance(cn, list) and cn:
class_names = [str(x) for x in cn]
if num_classes is None and eval_mode:
num_classes = 2 if eval_mode == "binary" else 3
return eval_mode, num_classes, class_names
def _collect_holdout_json_files(run_dir: Path, head: str) -> Dict[int, Path]:
fold_files: Dict[int, Path] = {}
# Fold-scoped folders
for folder_name in JSON_DIR_NAMES:
for fold_dir in run_dir.glob(f"fold*_{folder_name}"):
fold_match = re.search(r"fold(\d+)_", fold_dir.name)
if not fold_match:
continue
fold_idx = int(fold_match.group(1))
candidates = list(fold_dir.glob(f"epoch*_holdout_{head}.json"))
if not candidates:
candidates = list(fold_dir.glob(f"epoch*_{head}.json"))
if candidates:
candidates.sort(key=_epoch_from_name)
fold_files[fold_idx] = candidates[-1]
if fold_files:
return fold_files
# Fallback: unscoped roc_curves in run_dir (single-fold or in-progress)
for folder_name in JSON_DIR_NAMES:
base_dir = run_dir / folder_name
if not base_dir.exists():
continue
candidates = list(base_dir.glob(f"epoch*_holdout_{head}.json"))
if not candidates:
candidates = list(base_dir.glob(f"epoch*_{head}.json"))
if candidates:
candidates.sort(key=_epoch_from_name)
fold_files[0] = candidates[-1]
break
return fold_files
def _collect_validation_json_files(
holdout_files: Dict[int, Path], head: str
) -> Dict[int, Path]:
validation_files: Dict[int, Path] = {}
for fold_idx, holdout_path in holdout_files.items():
epoch = _epoch_from_name(holdout_path)
if epoch < 0:
continue
candidate = holdout_path.parent / f"epoch{epoch}_{head}.json"
if candidate.exists():
validation_files[fold_idx] = candidate
continue
# Fallback: try the same epoch under roc_curves (if holdout_best folder omitted it).
for folder_name in JSON_DIR_NAMES:
alt_dir = holdout_path.parent.parent / f"fold{fold_idx}_{folder_name}"
alt_candidate = alt_dir / f"epoch{epoch}_{head}.json"
if alt_candidate.exists():
validation_files[fold_idx] = alt_candidate
break
return validation_files
def _collect_holdout_from_validation_files(
validation_files: Dict[int, Path], head: str
) -> Dict[int, Path]:
holdout_files: Dict[int, Path] = {}
for fold_idx, val_path in validation_files.items():
epoch = _epoch_from_name(val_path)
if epoch < 0:
continue
candidate = val_path.parent / f"epoch{epoch}_holdout_{head}.json"
if candidate.exists():
holdout_files[fold_idx] = candidate
continue
for folder_name in ("roc_curves", "roc_curves_holdout_best"):
alt_dir = val_path.parent.parent / f"fold{fold_idx}_{folder_name}"
alt_candidate = alt_dir / f"epoch{epoch}_holdout_{head}.json"
if alt_candidate.exists():
holdout_files[fold_idx] = alt_candidate
break
return holdout_files
def _collect_best_json_files(run_dir: Path, head: str) -> Dict[int, Path]:
fold_files: Dict[int, Path] = {}
for fold_dir in run_dir.glob("fold*_roc_curves_best"):
fold_match = re.search(r"fold(\d+)_", fold_dir.name)
if not fold_match:
continue
fold_idx = int(fold_match.group(1))
candidates = list(fold_dir.glob(f"epoch*_{head}.json"))
if candidates:
candidates.sort(key=_epoch_from_name)
fold_files[fold_idx] = candidates[-1]
return fold_files
def _extract_curves(data: Dict) -> Dict[str, Tuple[List[float], List[float], float]]:
curves: Dict[str, Tuple[List[float], List[float], float]] = {}
per_class = data.get("per_class") if isinstance(data, dict) else None
if not isinstance(per_class, dict):
return curves
for cls, entry in per_class.items():
if not isinstance(entry, dict):
continue
fpr = entry.get("fpr")
tpr = entry.get("tpr")
auc_val = entry.get("auc")
if not isinstance(fpr, list) or not isinstance(tpr, list):
continue
try:
auc_f = float(auc_val) if auc_val is not None else float("nan")
except Exception:
auc_f = float("nan")
curves[str(cls)] = (fpr, tpr, auc_f)
return curves
def _derive_positive_from_class0(
curves_by_class: Dict[str, List[Tuple[int, List[float], List[float], float]]],
positive_class: int,
) -> None:
zero_key = "0"
if zero_key not in curves_by_class:
return
derived = []
for fold_idx, fpr0, tpr0, auc0 in curves_by_class.get(zero_key, []):
# The JSON for binary currently stores class-1 labels with class-0 scores,
# so invert the curve to recover the true class-1 ROC.
fpr1 = [1.0 - float(x) for x in fpr0]
tpr1 = [1.0 - float(x) for x in tpr0]
# Ensure increasing FPR for plotting.
if len(fpr1) > 1 and fpr1[0] > fpr1[-1]:
fpr1 = list(reversed(fpr1))
tpr1 = list(reversed(tpr1))
auc1 = 1.0 - auc0 if auc0 == auc0 else auc0
derived.append((fold_idx, fpr1, tpr1, auc1))
curves_by_class[str(positive_class)] = derived
def _needs_positive_derivation(
curves_by_class: Dict[str, List[Tuple[int, List[float], List[float], float]]],
positive_class: int,
) -> bool:
curves = curves_by_class.get(str(positive_class))
if not curves:
return True
for _, fpr, tpr, auc_val in curves:
if auc_val == auc_val and len(fpr) > 2 and len(tpr) > 2:
return False
return True
def _collect_prob_files(run_dir: Path, suffix: str) -> Dict[int, Dict[str, Path]]:
files: Dict[int, Dict[str, Path]] = {}
for y_file in run_dir.glob(f"fold*_y_true{suffix}.npy"):
fold_str = y_file.stem.split("_")[0].replace("fold", "")
try:
fold_idx = int(fold_str)
except ValueError:
continue
files.setdefault(fold_idx, {})["y_true"] = y_file
for head, key in HEAD_FILE_KEYS.items():
for p_file in run_dir.glob(f"fold*_probs_{key}{suffix}.npy"):
fold_str = p_file.stem.split("_")[0].replace("fold", "")
try:
fold_idx = int(fold_str)
except ValueError:
continue
files.setdefault(fold_idx, {})[head] = p_file
return files
def _load_array(path: Path) -> np.ndarray | None:
try:
return np.load(path)
except Exception:
return None
def _compute_binary_curve(
y_true: np.ndarray, probs: np.ndarray, positive_class: int
) -> Tuple[List[float], List[float], float] | None:
if probs.ndim == 1:
scores = probs
elif probs.ndim == 2 and probs.shape[1] > positive_class:
scores = probs[:, positive_class]
else:
return None
y_bin = (y_true == positive_class).astype(int)
if y_bin.sum() == 0 or y_bin.sum() == len(y_bin):
return None
fpr, tpr, _ = roc_curve(y_bin, scores)
auc_val = float(auc(fpr, tpr))
return fpr.tolist(), tpr.tolist(), auc_val
def _compute_multiclass_curves(
y_true: np.ndarray, probs: np.ndarray
) -> Dict[str, Tuple[List[float], List[float], float]]:
curves: Dict[str, Tuple[List[float], List[float], float]] = {}
if probs.ndim != 2:
return curves
num_classes = probs.shape[1]
for cls in range(num_classes):
y_bin = (y_true == cls).astype(int)
if y_bin.sum() == 0 or y_bin.sum() == len(y_bin):
continue
fpr, tpr, _ = roc_curve(y_bin, probs[:, cls])
curves[str(cls)] = (fpr.tolist(), tpr.tolist(), float(auc(fpr, tpr)))
return curves
def _plot_overlays(
curves_by_fold: List[Tuple[int, List[float], List[float], float]],
title: str,
out_path: Path,
) -> None:
fig, ax = plt.subplots(figsize=(6, 5))
for fold_idx, fpr, tpr, auc_val in curves_by_fold:
label = (
f"fold{fold_idx} AUC={auc_val:.3f}"
if auc_val == auc_val
else f"fold{fold_idx}"
)
ax.plot(fpr, tpr, lw=1.4, label=label)
ax.plot([0, 1], [0, 1], "k--", lw=1)
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(title)
ax.legend(loc="lower right", fontsize="small")
ax.grid(True, alpha=0.3, linestyle="--")
fig.tight_layout()
out_path.parent.mkdir(parents=True, exist_ok=True)
fig.savefig(out_path, dpi=170)
plt.close(fig)
def main() -> None:
run_dir = ANALYSIS_ROOT / RUN_ID
if not run_dir.exists():
raise SystemExit(f"Run not found: {run_dir}")
eval_mode, num_classes, class_names = _infer_run_info(run_dir)
is_binary = eval_mode == "binary" or num_classes == 2
def _plot_set(
label: str,
head: str,
json_files: Dict[int, Path],
out_dir: Path,
paired_files: Dict[int, Path] | None,
paired_suffix: str,
class_names: List[str] | None,
) -> None:
if not json_files:
return
curves_by_class: Dict[
str, List[Tuple[int, List[float], List[float], float]]
] = {}
paired_curves_by_class: Dict[
str, List[Tuple[int, List[float], List[float], float]]
] = {}
for fold_idx, path in sorted(json_files.items()):
if DEBUG:
print(f"[debug] {label} head={head} fold={fold_idx} json={path}")
data = _load_json(path)
curves = _extract_curves(data)
for cls, (fpr, tpr, auc_val) in curves.items():
curves_by_class.setdefault(cls, []).append(
(fold_idx, fpr, tpr, auc_val)
)
if paired_files:
p_path = paired_files.get(fold_idx)
if p_path is not None:
if DEBUG:
print(
f"[debug] {label} head={head} fold={fold_idx} paired_json={p_path}"
)
p_data = _load_json(p_path)
p_curves = _extract_curves(p_data)
for cls, (fpr, tpr, auc_val) in p_curves.items():
paired_curves_by_class.setdefault(cls, []).append(
(fold_idx, fpr, tpr, auc_val)
)
if _needs_positive_derivation(curves_by_class, POSITIVE_CLASS):
_derive_positive_from_class0(curves_by_class, POSITIVE_CLASS)
if paired_curves_by_class and _needs_positive_derivation(
paired_curves_by_class, POSITIVE_CLASS
):
_derive_positive_from_class0(paired_curves_by_class, POSITIVE_CLASS)
if PLOT_ALL_CLASSES:
classes = list(curves_by_class.keys())
else:
classes = (
[str(POSITIVE_CLASS)]
if str(POSITIVE_CLASS) in curves_by_class
else list(curves_by_class.keys())
)
if not classes:
return
for cls in classes:
fold_curves = curves_by_class.get(cls, [])
if not fold_curves:
continue
class_label = cls
if class_names is not None:
try:
idx = int(cls)
if 0 <= idx < len(class_names):
class_label = f"{cls} ({class_names[idx]})"
except Exception:
pass
title = f"{RUN_ID} {label} ROC — head={head} class={class_label}"
out_path = out_dir / f"{label}_{head}_class{cls}.png"
_plot_overlays(fold_curves, title, out_path)
print(f"[ok] {out_path}")
if paired_curves_by_class:
p_curves = paired_curves_by_class.get(cls, [])
if p_curves:
p_title = f"{RUN_ID} {label} {paired_suffix} ROC — head={head} class={class_label}"
p_path = out_dir / f"{label}_{head}_class{cls}_{paired_suffix}.png"
_plot_overlays(p_curves, p_title, p_path)
print(f"[ok] {p_path}")
def _plot_from_probs(
label: str,
head: str,
out_dir: Path,
suffix: str,
class_names: List[str] | None,
) -> None:
curves_by_class: Dict[
str, List[Tuple[int, List[float], List[float], float]]
] = {}
files = _collect_prob_files(run_dir, suffix)
if not files and suffix and ALLOW_FALLBACK_TO_VALIDATION:
files = _collect_prob_files(run_dir, "")
if files:
print(
"[warn] Holdout probability dumps not found; using validation probabilities instead."
)
if not files:
return
for fold_idx in sorted(files.keys()):
fold_files = files[fold_idx]
y_path = fold_files.get("y_true")
p_path = fold_files.get(head)
if y_path is None or p_path is None:
continue
y_true = _load_array(y_path)
probs = _load_array(p_path)
if y_true is None or probs is None:
continue
curves = _compute_multiclass_curves(y_true, probs)
for cls, payload in curves.items():
curves_by_class.setdefault(cls, []).append((fold_idx, *payload))
if not curves_by_class:
return
classes = list(curves_by_class.keys())
for cls in classes:
fold_curves = curves_by_class.get(cls, [])
if not fold_curves:
continue
class_label = cls
if class_names is not None:
try:
idx = int(cls)
if 0 <= idx < len(class_names):
class_label = f"{cls} ({class_names[idx]})"
except Exception:
pass
title = f"{RUN_ID} {label} ROC — head={head} class={class_label}"
out_path = out_dir / f"{label}_{head}_class{cls}.png"
_plot_overlays(fold_curves, title, out_path)
print(f"[ok] {out_path}")
any_holdout_json = False
if USE_JSON_ROC:
out_dir = run_dir / OUTPUT_SUBDIR
for head in HEADS:
files = _collect_holdout_json_files(run_dir, head)
if files:
any_holdout_json = True
paired = (
_collect_validation_json_files(files, head)
if PLOT_VALIDATION_FROM_HOLDOUT_EPOCH
else None
)
if is_binary and FORCE_PROBS_FOR_HOLDOUT_BINARY:
_plot_from_probs("holdout", head, out_dir, "_holdout", class_names)
else:
_plot_set(
"holdout",
head,
files,
out_dir,
paired,
"validation",
class_names,
)
if not any_holdout_json and DEBUG:
print("[debug] no JSON ROC files found; falling back to probs")
if PLOT_BEST_EPOCH:
best_out_dir = run_dir / BEST_OUTPUT_SUBDIR
for head in HEADS:
if is_binary and FORCE_PROBS_FOR_BEST_BINARY:
_plot_from_probs("best", head, best_out_dir, "", class_names)
continue
best_files = _collect_best_json_files(run_dir, head)
if best_files:
paired = (
_collect_holdout_from_validation_files(best_files, head)
if PLOT_HOLDOUT_FROM_BEST_EPOCH
else None
)
_plot_set(
"best",
head,
best_files,
best_out_dir,
paired,
"holdout",
class_names,
)
# Fallback to probs for holdout plots if JSON wasn't found.
if USE_JSON_ROC and any_holdout_json:
return
out_dir = run_dir / OUTPUT_SUBDIR
for head in HEADS:
curves_by_class: Dict[
str, List[Tuple[int, List[float], List[float], float]]
] = {}
suffix = "_holdout" if USE_HOLDOUT_PROBS else ""
files = _collect_prob_files(run_dir, suffix)
if not files and USE_HOLDOUT_PROBS and ALLOW_FALLBACK_TO_VALIDATION:
suffix = ""
files = _collect_prob_files(run_dir, suffix)
if files:
print(
"[warn] Holdout probability dumps not found; using validation probabilities instead."
)
if not files:
raise SystemExit(
"No saved probability dumps found. If you want holdout ROC curves, "
"run scripts/rebuild_run_best_plots.py with --use-holdout --overwrite "
"to generate fold*_y_true_holdout.npy and fold*_probs_*_holdout.npy files."
)
for fold_idx in sorted(files.keys()):
fold_files = files[fold_idx]
y_path = fold_files.get("y_true")
p_path = fold_files.get(head)
if y_path is None or p_path is None:
continue
y_true = _load_array(y_path)
probs = _load_array(p_path)
if y_true is None or probs is None:
continue
curves = _compute_multiclass_curves(y_true, probs)
for cls, payload in curves.items():
if payload is None:
continue
fpr, tpr, auc_val = payload
curves_by_class.setdefault(cls, []).append(
(fold_idx, fpr, tpr, auc_val)
)
if not curves_by_class:
continue
classes = sorted(curves_by_class.keys(), key=lambda x: (float(x), str(x)))
for cls in classes:
fold_curves = curves_by_class.get(cls, [])
if not fold_curves:
continue
class_label = cls
if class_names is not None:
try:
idx = int(cls)
if 0 <= idx < len(class_names):
class_label = f"{cls} ({class_names[idx]})"
except Exception:
pass
title = f"{RUN_ID} holdout ROC — head={head} class={class_label}"
out_path = out_dir / f"holdout_{head}_class{cls}.png"
_plot_overlays(fold_curves, title, out_path)
print(f"[ok] {out_path}")
if __name__ == "__main__":
main()
scripts/grid_search_analytics/param_perf_correlations.py
Executable
+514
View File
@@ -0,0 +1,514 @@
#!/usr/bin/env python3
"""
Analyze correlations between grid search parameters and performance metrics.
Example:
python scripts/grid_search_analytics/param_perf_correlations.py \
--analysis-dir analysis_data/grid_search \
--task binary \
--metric holdout_auc \
--top 30
"""
from __future__ import annotations
import argparse
import json
import math
import os
import sys
import time
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Tuple
DEFAULT_EXCLUDE_KEYS = {
"run_id",
"fold_metrics",
"best_metric",
"best_metric_mode",
"best_metric_mean",
"best_metric_std",
"eval_mode",
"n_splits",
"num_classes",
}
METRIC_MAP = {
"auc": "auc_fused",
"acc": "acc_fused",
"holdout_auc": "holdout_auc_fused",
"holdout_acc": "holdout_acc_fused",
}
def to_float(value: Optional[object]) -> Optional[float]:
if value is None:
return None
if isinstance(value, (int, float)) and not isinstance(value, bool):
num = float(value)
if math.isnan(num):
return None
return num
if not isinstance(value, str):
return None
value = value.strip()
if not value:
return None
try:
num = float(value)
except ValueError:
return None
if math.isnan(num):
return None
return num
def mean(values: List[float]) -> Optional[float]:
return (sum(values) / len(values)) if values else None
def read_json(path: Path) -> Optional[Dict[str, object]]:
if not path.exists():
return None
try:
data = json.loads(path.read_text())
except Exception:
return None
if not isinstance(data, dict):
return None
return data
def read_summary(run_dir: Path) -> Optional[Dict[str, object]]:
return read_json(run_dir / "summary.json")
def read_cli_args(run_dir: Path) -> Optional[Dict[str, object]]:
return read_json(run_dir / "cli_args.json")
def read_run_id(run_dir: Path, summary: Optional[Dict[str, object]]) -> str:
if summary:
rid = summary.get("run_id")
if isinstance(rid, str) and rid:
return rid
return run_dir.name
def task_from_summary(summary: Optional[Dict[str, object]]) -> Optional[str]:
if not summary:
return None
eval_mode = summary.get("eval_mode")
if isinstance(eval_mode, str):
mode = eval_mode.strip().lower()
if mode == "binary":
return "binary"
if mode in {"multiclass", "multi", "multi-class"}:
return "multiclass"
num_classes = summary.get("num_classes")
if isinstance(num_classes, (int, float)):
return "binary" if int(num_classes) <= 2 else "multiclass"
return None
def metric_from_stats(stats: Dict[str, object], metric: str) -> Optional[float]:
if metric.startswith("holdout_") and stats.get("holdout_best_monitor") == metric:
best_val = to_float(stats.get("holdout_best_so_far"))
if best_val is not None:
return best_val
return to_float(stats.get(metric))
def mean_metric(summary: Dict[str, object], metric: str) -> Optional[float]:
folds = summary.get("fold_metrics") or []
if not isinstance(folds, list) or not folds:
return None
values = []
for fold in folds:
stats = fold.get("stats") if isinstance(fold, dict) else None
if not isinstance(stats, dict):
return None
val = metric_from_stats(stats, metric)
if val is None:
return None
values.append(val)
return mean(values)
def flatten_config(data: Dict[str, object],
prefix: str = "",
exclude_keys: Optional[Iterable[str]] = None) -> Dict[str, object]:
out: Dict[str, object] = {}
excludes = set(exclude_keys or [])
for key, value in data.items():
if key in excludes or key.startswith("best_"):
continue
full_key = f"{prefix}{key}" if not prefix else f"{prefix}.{key}"
if isinstance(value, dict):
out.update(flatten_config(value, full_key, exclude_keys=excludes))
continue
if isinstance(value, list):
continue
out[full_key] = value
return out
def rankdata(values: List[float]) -> List[float]:
order = sorted(range(len(values)), key=lambda i: values[i])
ranks = [0.0] * len(values)
i = 0
while i < len(values):
j = i
while j + 1 < len(values) and values[order[j + 1]] == values[order[i]]:
j += 1
avg_rank = (i + j) / 2.0 + 1.0
for k in range(i, j + 1):
ranks[order[k]] = avg_rank
i = j + 1
return ranks
def pearson(x: List[float], y: List[float]) -> Optional[float]:
if len(x) != len(y) or len(x) < 2:
return None
mean_x = sum(x) / len(x)
mean_y = sum(y) / len(y)
num = sum((xi - mean_x) * (yi - mean_y) for xi, yi in zip(x, y))
den_x = sum((xi - mean_x) ** 2 for xi in x)
den_y = sum((yi - mean_y) ** 2 for yi in y)
if den_x <= 0 or den_y <= 0:
return None
return num / math.sqrt(den_x * den_y)
def spearman(x: List[float], y: List[float]) -> Optional[float]:
rx = rankdata(x)
ry = rankdata(y)
return pearson(rx, ry)
def correlation_ratio(categories: List[object], values: List[float]) -> Optional[float]:
if len(categories) != len(values) or len(values) < 2:
return None
overall = mean(values)
if overall is None:
return None
total = sum((v - overall) ** 2 for v in values)
if total <= 0:
return None
sums: Dict[object, List[float]] = {}
for cat, val in zip(categories, values):
sums.setdefault(cat, []).append(val)
between = 0.0
for vals in sums.values():
avg = mean(vals)
if avg is None:
continue
between += len(vals) * (avg - overall) ** 2
return math.sqrt(between / total)
def format_value(value: object) -> str:
if value is None:
return ""
if isinstance(value, bool):
return "true" if value else "false"
if isinstance(value, int):
return str(value)
if isinstance(value, float):
return f"{value:.6g}"
return str(value)
def format_metric(value: Optional[float]) -> str:
if value is None:
return ""
return f"{value:.4f}"
def format_table(rows: List[Dict[str, object]], columns: List[str]) -> str:
col_widths = {
col: max(len(col), max((len(str(row.get(col, ""))) for row in rows), default=0))
for col in columns
}
header = " | ".join(col.ljust(col_widths[col]) for col in columns)
divider = "-+-".join("-" * col_widths[col] for col in columns)
body = [
" | ".join(str(row.get(col, "")).ljust(col_widths[col]) for col in columns)
for row in rows
]
return "\n".join([header, divider, *body])
def render_progress(current: int, total: Optional[int], matched: int) -> str:
if total:
width = 30
filled = int(width * current / total)
bar = "#" * filled + "-" * (width - filled)
return f"[{bar}] {current}/{total} matched {matched}"
return f"Scanned {current} dirs, matched {matched}"
def iter_run_dirs(root: Path, shallow: bool, show_progress: bool) -> Iterable[Path]:
if shallow:
entries = [entry for entry in root.iterdir() if entry.is_dir()]
entries.sort(key=lambda p: p.name)
total = len(entries)
matched = 0
last_update = 0.0
for idx, entry in enumerate(entries, start=1):
if show_progress:
now = time.monotonic()
if now - last_update >= 0.1 or idx == total:
msg = render_progress(idx, total, matched)
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
last_update = now
if (entry / "summary.json").is_file():
matched += 1
yield entry
if show_progress:
print(file=sys.stderr)
return
matched = 0
scanned = 0
last_update = 0.0
for dirpath, dirnames, filenames in os.walk(root):
scanned += 1
if show_progress:
now = time.monotonic()
if now - last_update >= 0.2:
msg = render_progress(scanned, None, matched)
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
last_update = now
if "summary.json" in filenames:
matched += 1
yield Path(dirpath)
if show_progress:
msg = render_progress(scanned, None, matched)
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
print(file=sys.stderr)
def main() -> None:
ap = argparse.ArgumentParser(description="Correlate grid search parameters with performance.")
ap.add_argument("--analysis-dir", type=Path, default=Path("analysis_data/grid_search"),
help="Directory containing run subdirectories")
ap.add_argument("--task", choices=["binary", "multiclass", "all"], default="all",
help="Filter runs by task type (default: all)")
ap.add_argument("--metric", choices=sorted(METRIC_MAP.keys()), default="holdout_auc",
help="Performance metric to analyze (default: holdout_auc)")
ap.add_argument("--sort-by", choices=["score", "abs_rho", "rho", "r", "eta"], default="score",
help="Sorting key for results (default: score)")
ap.add_argument("--asc", action="store_true",
help="Sort ascending (default: descending)")
ap.add_argument("--desc", action="store_true",
help="Sort descending (default: descending)")
ap.add_argument("--top", type=int, default=30,
help="Limit output to top N parameters (default: 30)")
ap.add_argument("--params", default=None,
help="Comma-separated list of parameter keys to include")
ap.add_argument("--exclude", default=None,
help="Comma-separated list of parameter keys to exclude")
ap.add_argument("--min-count", type=int, default=10,
help="Minimum runs required to analyze a parameter (default: 10)")
ap.add_argument("--min-unique", type=int, default=2,
help="Minimum unique values required (default: 2)")
ap.add_argument("--match", default=None,
help="Only include run directories whose name contains this substring")
ap.add_argument("--shallow", action="store_true",
help="Only scan directories directly under analysis-dir")
ap.add_argument("--no-progress", action="store_true",
help="Disable progress output")
args = ap.parse_args()
if args.asc and args.desc:
raise SystemExit("Choose only one of --asc or --desc.")
root = args.analysis_dir
if not root.exists():
raise SystemExit(f"Analysis directory not found: {root}")
exclude_keys = set(DEFAULT_EXCLUDE_KEYS)
if args.exclude:
for item in args.exclude.split(","):
item = item.strip()
if item:
exclude_keys.add(item)
runs: List[Dict[str, object]] = []
values_by_key: Dict[str, List[object]] = {}
missing_summary = 0
unknown_task = 0
missing_cli = 0
metric_key = METRIC_MAP[args.metric]
for run_dir in iter_run_dirs(root, shallow=args.shallow, show_progress=not args.no_progress):
if args.match and args.match not in run_dir.name:
continue
summary = read_summary(run_dir)
if summary is None:
missing_summary += 1
continue
task_label = task_from_summary(summary)
if args.task != "all":
if task_label is None:
unknown_task += 1
continue
if task_label != args.task:
continue
metric_value = mean_metric(summary, metric_key)
if metric_value is None:
continue
cli_args = read_cli_args(run_dir)
if cli_args is None:
missing_cli += 1
config_source = cli_args if cli_args is not None else summary
config = flatten_config(config_source, exclude_keys=exclude_keys)
run_id = read_run_id(run_dir, summary)
runs.append({
"run_id": run_id,
"metric": metric_value,
"config": config,
})
for key, value in config.items():
values_by_key.setdefault(key, []).append(value)
if not runs:
print("No matching runs found.")
return
if args.params:
param_keys = [p.strip() for p in args.params.split(",") if p.strip()]
else:
param_keys = []
for key, values in values_by_key.items():
unique_values = {format_value(v) for v in values}
if len(unique_values) >= args.min_unique:
param_keys.append(key)
param_keys.sort()
rows: List[Dict[str, object]] = []
for key in param_keys:
values = []
metrics = []
for run in runs:
config = run.get("config", {})
if key not in config:
continue
values.append(config[key])
metrics.append(run["metric"])
if len(values) < args.min_count:
continue
unique_values = {format_value(v) for v in values}
if len(unique_values) < args.min_unique:
continue
numeric_values: List[float] = []
numeric_ok = True
for v in values:
num = to_float(v)
if num is None or isinstance(v, bool):
numeric_ok = False
break
numeric_values.append(num)
groups: Dict[object, List[float]] = {}
for val, metric in zip(values, metrics):
groups.setdefault(val, []).append(metric)
group_means = {k: mean(v) for k, v in groups.items()}
best_group = max(group_means.items(), key=lambda item: item[1] or float("-inf"))
worst_group = min(group_means.items(), key=lambda item: item[1] or float("inf"))
if numeric_ok and len(set(numeric_values)) >= 3:
rho = spearman(numeric_values, metrics)
r = pearson(numeric_values, metrics)
score = abs(rho) if rho is not None else None
row = {
"param": key,
"type": "numeric",
"n": len(values),
"distinct": len(unique_values),
"score": format_metric(score) if score is not None else "",
"rho": format_metric(rho),
"r": format_metric(r),
"best_value": format_value(best_group[0]),
"best_mean": format_metric(best_group[1]),
"worst_value": format_value(worst_group[0]),
"worst_mean": format_metric(worst_group[1]),
}
else:
eta = correlation_ratio(values, metrics)
score = eta
row = {
"param": key,
"type": "categorical",
"n": len(values),
"distinct": len(unique_values),
"score": format_metric(score) if score is not None else "",
"rho": "",
"r": "",
"best_value": format_value(best_group[0]),
"best_mean": format_metric(best_group[1]),
"worst_value": format_value(worst_group[0]),
"worst_mean": format_metric(worst_group[1]),
}
rows.append(row)
if not rows:
print("No parameters met the minimum requirements.")
return
def sort_key(row: Dict[str, object]) -> float:
raw = row.get(args.sort_by)
if isinstance(raw, str):
val = to_float(raw)
else:
val = to_float(raw)
if val is None:
return float("inf") if args.asc else float("-inf")
return float(val)
rows.sort(key=sort_key, reverse=not args.asc)
if args.top is not None:
rows = rows[:args.top]
header_lines = []
header_lines.append(f"Metric: {args.metric} (mean over folds)")
if args.task != "all":
header_lines.append(f"Task filter: {args.task}")
if args.match:
header_lines.append(f"Name filter: {args.match}")
header_lines.append(f"Runs: {len(runs)}")
if missing_cli:
header_lines.append(f"Missing cli_args: {missing_cli}")
if missing_summary or unknown_task:
header_lines.append(f"Skipped: {missing_summary} missing summary, {unknown_task} unknown task")
header_lines.append("")
print("\n".join(header_lines))
columns = [
"param",
"type",
"n",
"distinct",
"score",
"rho",
"r",
"best_value",
"best_mean",
"worst_value",
"worst_mean",
]
print(format_table(rows, columns))
if __name__ == "__main__":
main()
scripts/grid_search_analytics/rerun_grid_item.py
Executable
+212
View File
@@ -0,0 +1,212 @@
#!/usr/bin/env python3
"""Re-run a single grid-search configuration into analysis_data/re_runs."""
from __future__ import annotations
import argparse
import csv
import shutil
import subprocess
import sys
from pathlib import Path
from typing import Dict, List
# ---------------------------
# Config (edit in IDE)
# ---------------------------
RUN_ID = "20251129-0063" # fallback if --run-number is not provided
OUTPUT_RUN_ID = RUN_ID # fallback output run id
SHORTNAME = "re_runs" # output root under analysis_data/ and models/
GRID_PLAN = Path("analysis_data/grid_search/grid_plan.csv")
MANIFEST = Path("manifest.csv")
RUN_SCRIPT = Path("scripts/run_multifold.py")
REBUILD_SCRIPT = Path("scripts/rebuild_run_best_plots.py")
PLOT_HEADS = ["fused", "image", "metadata"]
USE_HOLDOUT_BEST_FOR_PLOTS = True
OVERWRITE_HOLDOUT_PROBS = True
ALLOW_EXISTING_RUN_DIR = False
def _parse_args() -> argparse.Namespace:
ap = argparse.ArgumentParser(description="Re-run a single grid-search item.")
ap.add_argument(
"--run-number",
type=str,
default=None,
help="Last 4 digits of run_id (e.g., 0063).",
)
ap.add_argument(
"--output-run-id",
type=str,
default=None,
help="Optional output run id; defaults to matched run_id.",
)
return ap.parse_args()
def _read_plan(path: Path) -> List[Dict[str, str]]:
if not path.exists():
raise FileNotFoundError(f"Grid plan not found: {path}")
with path.open(newline="") as fh:
reader = csv.DictReader(fh)
return list(reader)
def _find_row(rows: List[Dict[str, str]], run_id: str) -> Dict[str, str]:
for row in rows:
if row.get("run_id") == run_id:
return row
raise ValueError(f"run_id not found in grid plan: {run_id}")
def _resolve_run_id(rows: List[Dict[str, str]], run_number: str | None) -> str:
if not run_number:
return RUN_ID
run_number = str(run_number).strip()
if run_number.isdigit():
run_number = run_number.zfill(4)
matches = [
r.get("run_id", "")
for r in rows
if str(r.get("run_id", "")).endswith(f"-{run_number}")
]
if len(matches) == 1:
return matches[0]
if len(matches) > 1:
raise ValueError(
f"Multiple run_ids matched run-number '{run_number}': {matches[:5]}{' ...' if len(matches) > 5 else ''}"
)
raise ValueError(f"No run_id found ending with '-{run_number}'")
def _build_run_command(row: Dict[str, str], output_run_id: str) -> List[str]:
cmd = [
sys.executable,
str(RUN_SCRIPT),
"--backbone",
"resnet50",
"--fusion-mode",
"fused",
"--epochs",
"40",
"--batch-size",
"8",
"--img-crop-manifest",
str(MANIFEST),
"--img-crop-weights",
row["crop_weights"],
"--img-crop-normalize",
row["crop_normalize"],
"--eval_mode",
row["eval_mode"],
"--holdout-per-class",
"12",
"--run-id",
output_run_id,
"--shortname",
SHORTNAME,
]
if row.get("crop_tta") == "True":
cmd.append("--img-crop-tta")
loss_mode = row.get("loss_mode")
if loss_mode == "focal":
cmd.extend(["--focal-gamma", "2.0"])
elif loss_mode == "balanced":
cmd.append("--balanced-sampler")
thaw_mode = row.get("thaw_mode")
if thaw_mode == "gradual":
cmd.append("--gradual-thaw")
cmd.extend(["--thaw-ratio", "0.33"])
cmd.extend(["--thaw-start-epoch", "10"])
cmd.extend(["--thaw-target", "image"])
se_mode = row.get("se_mode")
if se_mode == "none":
cmd.append("--no-se")
else:
cmd.extend(["--se-reduction", "16"])
cmd.extend(["--se-reduction-tower", "16"])
cmd.extend(["--se-where", se_mode])
bridge_pre = row.get("se_bridge_pre_norm")
tower_pre = row.get("se_tower_pre_norm")
if bridge_pre == "True":
cmd.append("--se-pre-norm")
elif bridge_pre == "False":
cmd.append("--no-se-pre-norm")
if tower_pre == "True":
cmd.append("--se-pre-norm-tower")
elif tower_pre == "False":
cmd.append("--no-se-pre-norm-tower")
return cmd
def _swap_in_holdout_best(models_dir: Path) -> None:
for fold_dir in sorted(models_dir.glob("fold*")):
if not fold_dir.is_dir():
continue
holdout_best = fold_dir / "model_holdout_best.pt"
model_best = fold_dir / "model_best.pt"
if not holdout_best.exists():
print(f"[warn] {holdout_best} missing; skipping.")
continue
if model_best.exists():
backup = fold_dir / "model_best_from_train.pt"
if not backup.exists():
try:
shutil.copy2(model_best, backup)
except Exception:
pass
try:
shutil.copy2(holdout_best, model_best)
except Exception as exc:
print(f"[warn] failed to replace {model_best}: {exc}")
def _run_rebuild(run_dir: Path) -> None:
for head in PLOT_HEADS:
cmd = [
sys.executable,
str(REBUILD_SCRIPT),
"--run-dir",
str(run_dir),
"--head",
head,
"--use-holdout",
]
if OVERWRITE_HOLDOUT_PROBS:
cmd.append("--overwrite")
print("[rerun] Rebuilding holdout ROC plots:", " ".join(cmd))
subprocess.run(cmd, check=True)
def main() -> None:
args = _parse_args()
rows = _read_plan(GRID_PLAN)
run_id = _resolve_run_id(rows, args.run_number)
row = _find_row(rows, run_id)
output_run_id = args.output_run_id or run_id
run_dir = Path("analysis_data") / SHORTNAME / output_run_id
if run_dir.exists() and not ALLOW_EXISTING_RUN_DIR:
raise SystemExit(
f"Run directory already exists: {run_dir} (set ALLOW_EXISTING_RUN_DIR=True to reuse)"
)
cmd = _build_run_command(row, output_run_id=output_run_id)
print("[rerun] Launching:", " ".join(cmd))
subprocess.run(cmd, check=True)
models_dir = Path("models") / SHORTNAME / output_run_id
if USE_HOLDOUT_BEST_FOR_PLOTS:
print("[rerun] Swapping in holdout-best checkpoints for plotting.")
_swap_in_holdout_best(models_dir)
_run_rebuild(run_dir)
print(f"[rerun] Done. Outputs in {run_dir}")
if __name__ == "__main__":
main()
scripts/grid_search_analytics/rerun_grid_item_v2.py
+215
View File
@@ -0,0 +1,215 @@
#!/usr/bin/env python3
"""Re-run a single grid-search configuration using the V2 loader pipeline."""
from __future__ import annotations
import argparse
import csv
import shutil
import subprocess
import sys
from pathlib import Path
from typing import Dict, List
# ---------------------------
# Config (edit in IDE)
# ---------------------------
RUN_ID = "20251129-0063" # fallback if --run-number is not provided
OUTPUT_RUN_ID = RUN_ID # fallback output run id
SHORTNAME = "re_runs_v2" # output root under analysis_data/ and models/
GRID_PLAN = Path("analysis_data/grid_search/grid_plan.csv")
MANIFEST = Path("manifest.csv")
RUN_SCRIPT = Path("scripts/run_multifold_v2.py")
REBUILD_SCRIPT = Path("scripts/rebuild_run_best_plots.py")
PLOT_HEADS = ["fused", "image", "metadata"]
USE_HOLDOUT_BEST_FOR_PLOTS = True
OVERWRITE_HOLDOUT_PROBS = True
ALLOW_EXISTING_RUN_DIR = False
SAMPLE_MODE = "eye" # eye-level for parity with v1 grid runs
def _parse_args() -> argparse.Namespace:
ap = argparse.ArgumentParser(description="Re-run a single grid-search item with V2 loaders.")
ap.add_argument(
"--run-number",
type=str,
default=None,
help="Last 4 digits of run_id (e.g., 0063).",
)
ap.add_argument(
"--output-run-id",
type=str,
default=None,
help="Optional output run id; defaults to matched run_id.",
)
return ap.parse_args()
def _read_plan(path: Path) -> List[Dict[str, str]]:
if not path.exists():
raise FileNotFoundError(f"Grid plan not found: {path}")
with path.open(newline="") as fh:
reader = csv.DictReader(fh)
return list(reader)
def _find_row(rows: List[Dict[str, str]], run_id: str) -> Dict[str, str]:
for row in rows:
if row.get("run_id") == run_id:
return row
raise ValueError(f"run_id not found in grid plan: {run_id}")
def _resolve_run_id(rows: List[Dict[str, str]], run_number: str | None) -> str:
if not run_number:
return RUN_ID
run_number = str(run_number).strip()
if run_number.isdigit():
run_number = run_number.zfill(4)
matches = [
r.get("run_id", "")
for r in rows
if str(r.get("run_id", "")).endswith(f"-{run_number}")
]
if len(matches) == 1:
return matches[0]
if len(matches) > 1:
raise ValueError(
f"Multiple run_ids matched run-number '{run_number}': {matches[:5]}{' ...' if len(matches) > 5 else ''}"
)
raise ValueError(f"No run_id found ending with '-{run_number}'")
def _build_run_command(row: Dict[str, str], output_run_id: str) -> List[str]:
cmd = [
sys.executable,
str(RUN_SCRIPT),
"--backbone",
"resnet50",
"--fusion-mode",
"fused",
"--epochs",
"40",
"--batch-size",
"8",
"--img-crop-manifest",
str(MANIFEST),
"--img-crop-weights",
row["crop_weights"],
"--img-crop-normalize",
row["crop_normalize"],
"--eval_mode",
row["eval_mode"],
"--holdout-per-class",
"12",
"--run-id",
output_run_id,
"--shortname",
SHORTNAME,
"--sample-mode",
SAMPLE_MODE,
]
if row.get("crop_tta") == "True":
cmd.append("--img-crop-tta")
loss_mode = row.get("loss_mode")
if loss_mode == "focal":
cmd.extend(["--focal-gamma", "2.0"])
elif loss_mode == "balanced":
cmd.append("--balanced-sampler")
thaw_mode = row.get("thaw_mode")
if thaw_mode == "gradual":
cmd.append("--gradual-thaw")
cmd.extend(["--thaw-ratio", "0.33"])
cmd.extend(["--thaw-start-epoch", "10"])
cmd.extend(["--thaw-target", "image"])
se_mode = row.get("se_mode")
if se_mode == "none":
cmd.append("--no-se")
else:
cmd.extend(["--se-reduction", "16"])
cmd.extend(["--se-reduction-tower", "16"])
cmd.extend(["--se-where", se_mode])
bridge_pre = row.get("se_bridge_pre_norm")
tower_pre = row.get("se_tower_pre_norm")
if bridge_pre == "True":
cmd.append("--se-pre-norm")
elif bridge_pre == "False":
cmd.append("--no-se-pre-norm")
if tower_pre == "True":
cmd.append("--se-pre-norm-tower")
elif tower_pre == "False":
cmd.append("--no-se-pre-norm-tower")
return cmd
def _swap_in_holdout_best(models_dir: Path) -> None:
for fold_dir in sorted(models_dir.glob("fold*")):
if not fold_dir.is_dir():
continue
holdout_best = fold_dir / "model_holdout_best.pt"
model_best = fold_dir / "model_best.pt"
if not holdout_best.exists():
print(f"[warn] {holdout_best} missing; skipping.")
continue
if model_best.exists():
backup = fold_dir / "model_best_from_train.pt"
if not backup.exists():
try:
shutil.copy2(model_best, backup)
except Exception:
pass
try:
shutil.copy2(holdout_best, model_best)
except Exception as exc:
print(f"[warn] failed to replace {model_best}: {exc}")
def _run_rebuild(run_dir: Path) -> None:
for head in PLOT_HEADS:
cmd = [
sys.executable,
str(REBUILD_SCRIPT),
"--run-dir",
str(run_dir),
"--head",
head,
"--use-holdout",
]
if OVERWRITE_HOLDOUT_PROBS:
cmd.append("--overwrite")
print("[rerun] Rebuilding holdout ROC plots:", " ".join(cmd))
subprocess.run(cmd, check=True)
def main() -> None:
args = _parse_args()
rows = _read_plan(GRID_PLAN)
run_id = _resolve_run_id(rows, args.run_number)
row = _find_row(rows, run_id)
output_run_id = args.output_run_id or run_id
run_dir = Path("analysis_data") / SHORTNAME / output_run_id
if run_dir.exists() and not ALLOW_EXISTING_RUN_DIR:
raise SystemExit(
f"Run directory already exists: {run_dir} (set ALLOW_EXISTING_RUN_DIR=True to reuse)"
)
cmd = _build_run_command(row, output_run_id=output_run_id)
print("[rerun] Launching:", " ".join(cmd))
subprocess.run(cmd, check=True)
models_dir = Path("models") / SHORTNAME / output_run_id
if USE_HOLDOUT_BEST_FOR_PLOTS:
print("[rerun] Swapping in holdout-best checkpoints for plotting.")
_swap_in_holdout_best(models_dir)
_run_rebuild(run_dir)
print(f"[rerun] Done. Outputs in {run_dir}")
if __name__ == "__main__":
main()
scripts/grid_search_analytics/run_derived_analysis.py
+101
View File
@@ -0,0 +1,101 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
from pathlib import Path
from scripts.grid_search_analytics.derived_analysis import derived_analysis
def parse_args() -> argparse.Namespace:
ap = argparse.ArgumentParser(
description="Generate derived grid-search analytics artifacts (fusion/error + param-performance)."
)
ap.add_argument("--analysis-dir", default="analysis_data/grid_search")
ap.add_argument("--mode", choices=["binary", "multiclass"], default="multiclass")
ap.add_argument("--method", choices=["pearson", "spearman"], default="spearman")
ap.add_argument(
"--x-metric",
choices=["fusion_corrections", "fusion_corrections_per_opportunity"],
default="fusion_corrections_per_opportunity",
help="Fusion-correlation x-axis metric for summary bar plot.",
)
ap.add_argument(
"--cat-method",
choices=["eta", "anova", "kruskal"],
default="kruskal",
help="Categorical-test method for param-performance correlations.",
)
ap.add_argument("--top-n", type=int, default=None, help="Optional cap for per-run plots.")
ap.add_argument(
"--recompute",
action="store_true",
help="Recompute from run artifacts instead of preferring cached CSVs.",
)
ap.add_argument(
"--deep-scan",
action="store_true",
help="Scan nested directories instead of direct children only.",
)
return ap.parse_args()
def main() -> int:
args = parse_args()
analysis = derived_analysis(
Path(args.analysis_dir),
classification_mode=args.mode,
)
shallow = not args.deep_scan
existing = not args.recompute
analysis.identify_fusion_corrections(shallow=shallow, existing=existing)
analysis.populate_primary_metrics(shallow=shallow, existing=existing)
analysis.write_fusion_corrections()
analysis.write_fusion_errors()
analysis.write_primary_metrics()
analysis.plot_fusion_corrections_errors(
shallow=shallow, existing=existing, top_n=args.top_n
)
analysis.plot_conf_delta_boxplot(
shallow=shallow, existing=existing, top_n=args.top_n
)
corr_df = analysis.param_performance_correlations(
shallow=shallow,
existing=existing,
method=args.method,
cat_method=args.cat_method,
)
analysis.plot_param_perf_corr_panels(corr_df)
corr_acc = analysis.fusion_corrections_correlation(
method=args.method, metric_type="acc"
)
corr_auc = analysis.fusion_corrections_correlation(
method=args.method, metric_type="auc"
)
try:
analysis.plot_fusion_perf_summary(
corr_acc,
corr_auc,
method=args.method,
x_metric=args.x_metric,
)
except RuntimeError:
analysis.plot_fusion_perf_summary(
corr_acc,
corr_auc,
method=args.method,
x_metric="fusion_corrections",
)
print(f"Done. Outputs written under: {Path(args.analysis_dir) / 'plots'}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/legacy/classic_ml_roc.py
Executable
+326
View File
@@ -0,0 +1,326 @@
#!/usr/bin/env python3
"""Run classical ML models and plot *combined* ROC curves (multimodel overlays).
Keeps your original workflow for folds/tests exactly the same.
Only changes: collects predictions per test and makes:
• One ROC plot per class (OvR), overlaying all models
• One binary ROC plot (Healthy vs Glaucoma), overlaying all models
"""
import os
import re
from pathlib import Path
from typing import Iterable, List, Tuple, Dict, Optional
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler, label_binarize
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.metrics import roc_curve, auc
from classes import build_papila_clinical
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
SPLIT_ROOT = Path("HelpCode/kfold")
TRUST_INDEX_COL = False
# ---------------------------------------------------------------------------
# Feature matrix
# ---------------------------------------------------------------------------
def build_feature_matrix(clinical):
df = clinical.df.copy()
scalars = ["Age", "dioptre_1", "dioptre_2", "astigmatism", "Pachymetry", "Axial_Length", "IOP_corr"]
cats = ["Gender", "Phakic/Pseudophakic"]
X = pd.concat([df[scalars], pd.get_dummies(df[cats].astype("category"), drop_first=False, prefix=cats)], axis=1)
y = df[clinical.label_col].astype(int).values
return X, y, scalars, df # X keeps NaNs; we impute per-fold
# ---------------------------------------------------------------------------
# Models with tuned hyper-parameters (unchanged)
# ---------------------------------------------------------------------------
def make_models() -> Dict[str, Pipeline]:
return {
"LogReg": Pipeline([
("scaler", StandardScaler()),
("clf", LogisticRegression(
C=1,
class_weight="balanced",
max_iter=200,
solver="lbfgs",
multi_class="auto")),
]),
"kNN": Pipeline([
("scaler", StandardScaler()),
("clf", KNeighborsClassifier(
n_neighbors=11, weights="distance")),
]),
"RF": Pipeline([
("clf", RandomForestClassifier(n_estimators=200, max_depth=8,
min_samples_split=4, random_state=42)),
]),
"SVM": Pipeline([
("scaler", StandardScaler()),
("clf", SVC(C=10, kernel="rbf", gamma=0.1, probability=True)),
]),
}
# ---------------------------------------------------------------------------
# Split helpers copied from paper_clinical_baselines_official.py (unchanged)
# ---------------------------------------------------------------------------
_FNAME_RE = re.compile(r"RET\s*(\d+)\s*([Oo][DSs])\.jpg$", re.IGNORECASE)
def _read_sheet_any(p: Path) -> pd.DataFrame:
if p.suffix.lower() == ".xlsx":
return pd.read_excel(p)
if p.suffix.lower() == ".csv":
return pd.read_csv(p)
if p.suffix.lower() == ".txt":
lines = [ln.strip() for ln in p.read_text(encoding="utf-8", errors="ignore").splitlines() if ln.strip()]
return pd.DataFrame({"filename": lines})
raise ValueError(f"Unsupported split file type: {p.suffix}")
def _normcols(cols: List[str]) -> Dict[str, str]:
def norm(s: str) -> str:
return re.sub(r"[^a-z0-9]", "", s.lower())
return {norm(c): c for c in cols}
def _parse_fname_to_pid_eye(fname: str) -> Optional[Tuple[int, str]]:
base = os.path.basename(str(fname))
m = _FNAME_RE.search(base.replace(" ", ""))
if not m:
return None
return int(m.group(1)), m.group(2).upper()
def _rows_from_sheet(sheet: pd.DataFrame, df_master: pd.DataFrame) -> List[int]:
cols = _normcols(list(sheet.columns))
if "filename" in cols:
fn_col = cols["filename"]
lookup: Dict[str, List[int]] = {}
for i, (pid, eye) in enumerate(zip(df_master["Patient ID"].astype(int), df_master["eyeID"].astype(str))):
lookup.setdefault(f"{pid}|{eye.upper()}", []).append(i)
rows: List[int] = []
for fn in sheet[fn_col].astype(str).tolist():
pe = _parse_fname_to_pid_eye(fn)
if pe is None:
continue
pid, eye = pe
rows.extend(lookup.get(f"{pid}|{eye}", []))
return rows
if "patientid" in cols and "eyeid" in cols:
pid_col, eye_col = cols["patientid"], cols["eyeid"]
lookup = {}
for i, (pid, eye) in enumerate(zip(df_master["Patient ID"].astype(int), df_master["eyeID"].astype(str))):
lookup.setdefault(f"{pid}|{eye.upper()}", []).append(i)
rows = []
for pid, eye in zip(sheet[pid_col], sheet[eye_col]):
rows.extend(lookup.get(f"{int(pid)}|{str(eye).upper()}", []))
return rows
if TRUST_INDEX_COL and "index" in cols:
idx = sheet[cols["index"]].astype(int).tolist()
n = len(df_master)
return [i for i in idx if 0 <= i < n]
raise RuntimeError("Split sheet missing usable columns")
def _pair_train_test_files(dir_train: Path, dir_test: Path) -> List[Tuple[Path, Path]]:
def fold_key(p: Path) -> str:
m = re.search(r"(\d+)", p.stem)
return m.group(1) if m else p.stem.lower()
trains = sorted([p for p in dir_train.iterdir() if p.is_file() and p.suffix.lower() in (".xlsx", ".csv", ".txt")], key=fold_key)
tests = sorted([p for p in dir_test.iterdir() if p.is_file() and p.suffix.lower() in (".xlsx", ".csv", ".txt")], key=fold_key)
return [(trains[i], tests[i]) for i in range(min(len(trains), len(tests)))]
def iter_official_folds_xlsx(clinical, split_root: Path, test_name: str) -> Iterable[Tuple[pd.DataFrame, pd.DataFrame]]:
df_master = clinical.df.copy()
test_dir = split_root / test_name
dir_train = test_dir / "Train"
dir_test = test_dir / "Test"
if not dir_train.exists() or not dir_test.exists():
raise FileNotFoundError(f"Expected: {dir_train} and {dir_test}")
for train_file, test_file in _pair_train_test_files(dir_train, dir_test):
sh_tr, sh_te = _read_sheet_any(train_file), _read_sheet_any(test_file)
tr_rows, te_rows = _rows_from_sheet(sh_tr, df_master), _rows_from_sheet(sh_te, df_master)
tr_df, te_df = df_master.iloc[tr_rows].copy(), df_master.iloc[te_rows].copy()
yield tr_df, te_df
# ---------------------------------------------------------------------------
# Utilities (unchanged)
# ---------------------------------------------------------------------------
def _prepare_fold_X(X: pd.DataFrame, scalars: List[str], tr_idx: np.ndarray, te_idx: np.ndarray):
Xtr, Xte = X.iloc[tr_idx].copy(), X.iloc[te_idx].copy()
med = Xtr[scalars].median(numeric_only=True)
Xtr[scalars] = Xtr[scalars].fillna(med)
Xte[scalars] = Xte[scalars].fillna(med)
return Xtr.values.astype(np.float32), Xte.values.astype(np.float32)
# ---------------------------------------------------------------------------
# NEW: combined plotting helpers (multimodel overlays)
# ---------------------------------------------------------------------------
def _plot_multiclass_overlay(y_true: np.ndarray, prob_dict: Dict[str, np.ndarray], out_dir: Path, test_tag: str):
"""One figure per class (OvR), overlaying all models."""
n_classes = next(iter(prob_dict.values())).shape[1]
class_names = [f"Class{k}" for k in range(n_classes)]
y_bin = label_binarize(y_true, classes=list(range(n_classes)))
for k in range(n_classes):
fig, ax = plt.subplots(figsize=(6, 5))
for model_name, proba in prob_dict.items():
fpr, tpr, _ = roc_curve(y_bin[:, k], proba[:, k])
auc_val = auc(fpr, tpr)
ax.plot(fpr, tpr, lw=1.8, label=f"{model_name} (AUC={auc_val:.3f})")
ax.plot([0, 1], [0, 1], "k--", lw=1)
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(f"{class_names[k]} vs Rest — {test_tag}")
ax.legend(loc="lower right")
ax.grid(True, alpha=0.3, linestyle="--")
fig.tight_layout()
fig.savefig(out_dir / f"{test_tag}_{class_names[k]}.png", dpi=170)
plt.close(fig)
def _plot_binary_overlay(y_true: np.ndarray, prob1d_dict: Dict[str, np.ndarray], out_dir: Path, test_tag: str):
"""One figure (Healthy vs Glaucoma), overlaying all models. Assumes y_true ∈ {0,1}."""
fig, ax = plt.subplots(figsize=(6, 5))
any_curve = False
for model_name, scores in prob1d_dict.items():
if scores.size == 0:
continue
fpr, tpr, _ = roc_curve(y_true, scores, pos_label=1)
auc_val = auc(fpr, tpr)
ax.plot(fpr, tpr, lw=1.8, label=f"{model_name} (AUC={auc_val:.3f})")
any_curve = True
ax.plot([0, 1], [0, 1], "k--", lw=1)
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(f"Binary Healthy vs Glaucoma — {test_tag}")
if any_curve:
ax.legend(loc="lower right")
ax.grid(True, alpha=0.3, linestyle="--")
fig.tight_layout()
fig.savefig(out_dir / f"{test_tag}_binary.png", dpi=170)
plt.close(fig)
# ---------------------------------------------------------------------------
# Main (same folds/tests flow; only result collation & plotting changed)
# ---------------------------------------------------------------------------
def main():
clinical = build_papila_clinical(
image_dir="Papila/FundusImages",
clinical_dir="Papila/ClinicalData",
label_col="Diagnosis",
cat_cols=["Gender", "Phakic/Pseudophakic"],
)
X, y, scalars, _ = build_feature_matrix(clinical)
models = make_models()
out_dir = Path("analysis_data/roc_baselines")
out_dir.mkdir(parents=True, exist_ok=True)
tests = [
("Test 3", False), ("Test 4", True)
] if (SPLIT_ROOT / "Test 3").exists() else [
("Test 1", False), ("Test 2", True)
]
for test_name, is_binary in tests:
# Collect per-model probabilities following your original per-model loop.
# For multiclass: dict[model] -> (N, C)
# For binary: dict[model] -> (N,) (probability of class 1)
prob_dict_multi: Dict[str, np.ndarray] = {}
prob_dict_bin: Dict[str, np.ndarray] = {}
y_ref_multi: Optional[np.ndarray] = None
y_ref_bin: Optional[np.ndarray] = None
for model_name, model in models.items():
y_all: List[np.ndarray] = []
p_all: List[np.ndarray] = []
for fold_idx, (train_df, test_df) in enumerate(iter_official_folds_xlsx(clinical, SPLIT_ROOT, test_name), 1):
# Keep your exact masking/handling
dup_rows = set(train_df.index).intersection(set(test_df.index))
shared_pids = set(train_df["Patient ID"]).intersection(set(test_df["Patient ID"]))
if test_name in ("Test 1", "Test 2") and shared_pids:
train_df = train_df[~train_df["Patient ID"].isin(shared_pids)].copy()
dup_rows = set(train_df.index).intersection(set(test_df.index))
shared_pids = set(train_df["Patient ID"]).intersection(set(test_df["Patient ID"]))
tr_idx, te_idx = train_df.index.values, test_df.index.values
if is_binary:
# original binary handling: drop Suspects on both sets
mask_tr = np.isin(y[tr_idx], [0, 1])
mask_te = np.isin(y[te_idx], [0, 1])
if not mask_tr.any() or not mask_te.any():
# skip empty fold (keeps behavior safe without changing fold logic)
continue
Xtr, Xte = _prepare_fold_X(X, scalars, tr_idx[mask_tr], te_idx[mask_te])
ytr, yte = y[tr_idx][mask_tr], y[te_idx][mask_te]
else:
Xtr, Xte = _prepare_fold_X(X, scalars, tr_idx, te_idx)
ytr, yte = y[tr_idx], y[te_idx]
# Fit and score (unchanged approach)
model.fit(Xtr, ytr)
if is_binary:
if hasattr(model[-1], "predict_proba"):
prob = model.predict_proba(Xte)[:, 1]
else:
dec = model.decision_function(Xte)
prob = 1.0 / (1.0 + np.exp(-dec)) if np.ptp(dec) > 0 else np.full_like(dec, 0.5)
y_all.append(yte)
p_all.append(prob)
else:
if hasattr(model[-1], "predict_proba"):
prob = model.predict_proba(Xte)
else:
dec = model.decision_function(Xte)
if dec.ndim == 1:
dec = np.stack([-dec, dec], axis=1)
e = np.exp(dec - dec.max(axis=1, keepdims=True))
prob = e / e.sum(axis=1, keepdims=True)
y_all.append(yte)
p_all.append(prob)
if not y_all:
# No valid folds for this model under this test (e.g., all-bad after mask); skip
continue
y_cat = np.concatenate(y_all)
p_cat = np.concatenate(p_all)
if is_binary:
# Store 1D scores per model
prob_dict_bin[model_name] = p_cat
if y_ref_bin is None:
y_ref_bin = y_cat
else:
# Align lengths defensively (should match in normal use)
n = min(len(y_ref_bin), len(y_cat))
y_ref_bin = y_ref_bin[:n]
prob_dict_bin[model_name] = prob_dict_bin[model_name][:n]
else:
# Store (N, C) per model
prob_dict_multi[model_name] = p_cat
if y_ref_multi is None:
y_ref_multi = y_cat
else:
# Align lengths defensively (should match in normal use)
n = min(len(y_ref_multi), len(y_cat))
y_ref_multi = y_ref_multi[:n]
prob_dict_multi[model_name] = prob_dict_multi[model_name][:n, :]
tag = test_name.replace(" ", "")
# Produce overlays
if prob_dict_multi and y_ref_multi is not None:
_plot_multiclass_overlay(y_ref_multi, prob_dict_multi, out_dir, tag)
if prob_dict_bin and y_ref_bin is not None:
_plot_binary_overlay(y_ref_bin, prob_dict_bin, out_dir, tag)
if __name__ == "__main__":
main()
scripts/legacy/eval_papila_classifier.py
Executable
+384
View File
@@ -0,0 +1,384 @@
"""Evaluate REFUGE-trained classifier on Papila images using UNet crops."""
from __future__ import annotations
import argparse
import csv
from pathlib import Path
from typing import Dict, List, Optional, Sequence, Set
import numpy as np
import torch
from torch.utils.data import DataLoader
from tqdm import tqdm
from PIL import Image, ImageDraw
import sys
REPO_ROOT = Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.refuge_preprocessing import RefugePreprocessing, RefugeSample
from classes.refuge_segmentation import RefugeSegmentation
from classes.refuge_classification import (
RefugeClassification,
RefugeClassificationDataset,
RefugeClassificationRecord,
UNetGeometryProvider,
_default_image_transform,
_geometry_from_mask,
)
from classes.backbones import BACKBONES, load_backbone_weights
from classes.unet_segmenter import UNetSegmenter
from classes.papila_builders import build_papila_clinical
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Evaluate classifier on Papila with UNet crops")
parser.add_argument("--filtered-metrics", type=Path, required=True, help="CSV of Papila samples with acceptable Dice")
parser.add_argument("--segmenter-manifest", type=Path, required=True, help="Manifest used to train the UNet segmenter")
parser.add_argument("--segmenter-weights", type=Path, required=True, help="Path to trained UNet weights (best.pt)")
parser.add_argument("--classifier-weights", type=Path, required=False, help="Path to classifier checkpoint (refuge_classifier_best.pt)")
parser.add_argument("--refuge-root", type=Path, default=Path("REFUGE"))
parser.add_argument("--image-dir", type=Path, default=Path("Papila/FundusImages"))
parser.add_argument("--clinical-dir", type=Path, default=Path("Papila/ClinicalData"))
parser.add_argument("--label-col", type=str, default="Diagnosis", help="Column name holding Papila labels")
parser.add_argument(
"--positive-labels",
nargs="*",
default=["glaucoma", "glaucoma suspect", "suspect"],
help="Values treated as glaucoma-positive when labels are non-numeric",
)
parser.add_argument("--dice-threshold", type=float, default=0.01, help="Minimum Dice (disc or cup) to keep a sample")
parser.add_argument("--segmenter-threshold", type=float, default=0.5, help="Probability threshold for UNet geometry")
parser.add_argument("--segmenter-normalize", choices=["none", "imagenet", "per_image"], default="per_image")
parser.add_argument("--segmenter-tta", action="store_true", help="Enable TTA (H/V flips) when deriving geometry")
parser.add_argument("--crop-scale", type=float, default=2.5)
parser.add_argument("--crop-size", type=int, default=224)
parser.add_argument("--batch-size", type=int, default=32)
parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
parser.add_argument("--output", type=Path, default=None, help="Optional CSV to store per-sample probabilities")
parser.add_argument(
"--cache-dir",
type=Path,
default=Path("analysis_data/classifier_cache"),
help="Directory to reuse classifier preprocessing cache",
)
parser.add_argument(
"--use-gt-masks",
action="store_true",
help="Use ground truth Papila contours instead of UNet predictions",
)
parser.add_argument(
"--gt-contours-dir",
type=Path,
default=Path("Papila/ExpertsSegmentations/Contours"),
help="Directory containing Papila contour text files",
)
parser.add_argument(
"--backbone",
type=str,
default=None,
help="Optional backbone name (e.g. inception_v3, densenet121). Requires matching classifier weights.",
)
return parser.parse_args()
def load_allowed_ids(path: Path, dice_threshold: float) -> Set[str]:
allowed: Set[str] = set()
with path.open(newline="") as fp:
reader = csv.DictReader(fp)
for row in reader:
sample_id = row.get("sample_id")
if not sample_id or sample_id == "__mean__":
continue
try:
disc = float(row.get("dice_disc", "nan"))
cup = float(row.get("dice_cup", "nan"))
except ValueError:
continue
if disc < dice_threshold and cup < dice_threshold:
continue
allowed.add(sample_id)
return allowed
def build_papila_samples(
image_dir: Path,
clinical_dir: Path,
label_col: str,
positive_labels: Sequence[str],
allowed_ids: Set[str],
) -> List[RefugeSample]:
clinical = build_papila_clinical(
image_dir=str(image_dir),
clinical_dir=str(clinical_dir),
label_col=label_col,
cat_cols=[],
)
positives = {lbl.lower() for lbl in positive_labels}
samples: Dict[str, RefugeSample] = {}
for _, row in clinical.df.iterrows():
image_path = clinical.get_image_path(row)
sample_id = f"papila_{image_path.stem}"
if sample_id not in allowed_ids or sample_id in samples:
continue
value = row.get(label_col)
if value is None or (isinstance(value, float) and np.isnan(value)):
continue
label: Optional[int]
try:
label_int = int(value)
if label_int == 2:
continue
label = 1 if label_int > 0 else 0
except (TypeError, ValueError):
label = 1 if str(value).strip().lower() in positives else 0
samples[sample_id] = RefugeSample(
sample_id=sample_id,
dataset="papila",
split="eval",
image_path=Path(image_path),
label=label,
device=None,
mask_path=None,
fovea_coord=None,
)
return list(samples.values())
def load_contour(path: Path) -> np.ndarray:
coords = np.loadtxt(path)
if coords.ndim == 1:
coords = coords.reshape(-1, 2)
return coords
def contour_to_mask(coords: np.ndarray, size: Sequence[int]) -> np.ndarray:
if coords is None or coords.size == 0:
return np.zeros((size[1], size[0]), dtype=np.uint8)
img = Image.new("L", size, 0)
draw = ImageDraw.Draw(img)
points = [tuple(map(float, pt)) for pt in coords]
draw.polygon(points, outline=1, fill=1)
return np.array(img, dtype=np.uint8)
class PapilaGTGeometryProvider:
def __init__(self, contours_dir: Path) -> None:
self.contours_dir = contours_dir
def _pick(self, base: str, kind: str) -> Optional[Path]:
for exp in ("exp2", "exp1"):
cand = self.contours_dir / f"{base}_{kind}_{exp}.txt"
if cand.exists():
return cand
return None
def __call__(self, sample: RefugeSample, scale: float):
base = Path(sample.image_path).stem
disc_path = self._pick(base, "disc")
cup_path = self._pick(base, "cup")
if disc_path is None or cup_path is None:
raise RuntimeError(f"Missing ground-truth contours for {sample.sample_id}")
image = Image.open(sample.image_path).convert("RGB")
disc_coords = load_contour(disc_path)
cup_coords = load_contour(cup_path)
disc_mask = contour_to_mask(disc_coords, image.size)
cup_mask = contour_to_mask(cup_coords, image.size)
cup_mask = ((cup_mask > 0) & (disc_mask > 0)).astype(np.uint8)
geom = _geometry_from_mask(disc_mask, scale)
return geom, disc_mask.astype(np.uint8), cup_mask.astype(np.uint8)
def build_backbone(name: Optional[str]) -> Optional[torch.nn.Module]:
if not name:
return None
key = name.lower()
if key not in BACKBONES:
raise ValueError(f"Unknown backbone '{name}'. Available: {', '.join(sorted(BACKBONES.keys()))}")
spec = BACKBONES[key]
model = spec.ctor(weights=spec.weights_default)
out_dim, model = spec.strip(model)
setattr(model, "_feature_dim", out_dim)
if key == "refugelike":
load_backbone_weights(key, model)
return model
def evaluate_records(
clf: RefugeClassification,
records: Sequence[RefugeClassificationRecord],
device: str,
batch_size: int,
) -> Dict[str, float]:
dataset = RefugeClassificationDataset(
records,
transform=clf.eval_transform,
polar_transform=clf.polar_transform,
size=clf.crop_size,
)
loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=0)
clf.backbone.to(device).eval()
clf.classifier_head.to(device).eval()
preds: List[float] = []
targets: List[int] = []
with torch.no_grad():
for batch in tqdm(loader, desc="Papila Eval", leave=False, unit="batch"):
images = batch["image"].to(device)
polars = batch["polar"].to(device)
extra_feats = batch["features"].to(device)
labels = batch["label"].cpu().numpy().tolist()
feats_img = clf.backbone(images)
feats = feats_img
if clf.use_polar:
feats_polar = clf.backbone(polars)
feats = torch.cat([feats, feats_polar], dim=1)
if clf.extra_feature_dim > 0:
feats = torch.cat([feats, extra_feats], dim=1)
logits = clf.classifier_head(feats)
probs = torch.softmax(logits, dim=1)[:, 1].cpu().numpy().tolist()
preds.extend(probs)
targets.extend(labels)
metrics: Dict[str, float] = {"count": float(len(targets))}
unique_labels = set(targets)
if len(unique_labels) >= 2:
metrics["auc"] = float(torchmetrics_auc(targets, preds))
else:
metrics["auc"] = float("nan")
preds_bin = [1 if p >= 0.5 else 0 for p in preds]
accuracy = sum(int(p == t) for p, t in zip(preds_bin, targets)) / max(1, len(targets))
metrics["accuracy"] = float(accuracy)
metrics["mean_prob"] = float(np.mean(preds)) if preds else float("nan")
metrics["labels_pos"] = float(sum(targets))
if preds:
metrics["probs_std"] = float(np.std(preds))
return metrics
def torchmetrics_auc(targets: Sequence[int], preds: Sequence[float]) -> float:
try:
from sklearn.metrics import roc_auc_score
except ImportError as exc:
raise RuntimeError("scikit-learn is required to compute AUC") from exc
return float(roc_auc_score(targets, preds))
def main() -> None:
args = parse_args()
device = args.device
allowed_ids = load_allowed_ids(args.filtered_metrics, args.dice_threshold)
if not allowed_ids:
raise SystemExit("No Papila samples passed the Dice threshold.")
papila_samples = build_papila_samples(
args.image_dir,
args.clinical_dir,
args.label_col,
args.positive_labels,
allowed_ids,
)
if not papila_samples:
raise SystemExit("No Papila samples with labels matched the filtered metrics.")
cache_dir = args.cache_dir
if args.use_gt_masks and cache_dir is not None:
cache_dir = cache_dir / "gt"
if args.use_gt_masks:
geometry_provider = PapilaGTGeometryProvider(args.gt_contours_dir)
segmenter = None
else:
segmenter = UNetSegmenter(
manifest_path=args.segmenter_manifest,
device=device,
normalize=args.segmenter_normalize,
)
seg_state = torch.load(args.segmenter_weights, map_location=device)
seg_state_dict = seg_state.get("model", seg_state)
segmenter.model.load_state_dict(seg_state_dict)
segmenter.model.to(device)
geometry_provider = UNetGeometryProvider(
segmenter=segmenter,
threshold=args.segmenter_threshold,
tta=args.segmenter_tta,
)
pre = RefugePreprocessing(args.refuge_root)
dummy_seg = RefugeSegmentation(pre)
backbone = build_backbone(args.backbone)
clf = RefugeClassification(
pre,
dummy_seg,
geometry_fn=geometry_provider,
cache_dir=cache_dir,
backbone=backbone,
)
clf.crop_scale = args.crop_scale
clf.crop_size = args.crop_size
clf.eval_transform = _default_image_transform(args.crop_size)
clf.ttt_transform = clf.eval_transform
if args.classifier_weights is not None:
clf_state = torch.load(args.classifier_weights, map_location=device)
clf.backbone.load_state_dict(clf_state["backbone"])
clf.classifier_head.load_state_dict(clf_state["classifier"])
clf.rotation_head.load_state_dict(clf_state["rotation"])
if "feature_reg" in clf_state and getattr(clf, "feature_reg_head", None) is not None:
clf.feature_reg_head.load_state_dict(clf_state["feature_reg"])
records = clf.build_records_for_samples(
papila_samples,
crop_scale=args.crop_scale,
progress_prefix="papila_eval",
)
if not records:
raise SystemExit("Unable to build any records; check geometry predictions or labels.")
metrics = evaluate_records(clf, records, device=device, batch_size=args.batch_size)
print(f"Samples evaluated: {int(metrics['count'])}")
print(f"AUC: {metrics['auc']:.4f}" if not np.isnan(metrics['auc']) else "AUC: NaN")
print(f"Accuracy @0.5: {metrics['accuracy']:.4f}")
print(f"Mean glaucoma prob: {metrics['mean_prob']:.4f}")
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
with args.output.open("w", newline="") as fp:
writer = csv.writer(fp)
writer.writerow(["sample_id", "prob_glaucoma", "label"])
clf.backbone.eval()
clf.classifier_head.eval()
dataset = RefugeClassificationDataset(
records,
transform=clf.eval_transform,
polar_transform=clf.polar_transform,
size=clf.crop_size,
)
loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=False, num_workers=0)
with torch.no_grad():
for batch in tqdm(loader, desc="Papila Output", leave=False, unit="batch"):
images = batch["image"].to(device)
polars = batch["polar"].to(device)
extra_feats = batch["features"].to(device)
ids = batch["sample_id"]
labels = batch["label"].tolist()
feats_img = clf.backbone(images)
feats = feats_img
if clf.use_polar:
feats_polar = clf.backbone(polars)
feats = torch.cat([feats, feats_polar], dim=1)
if clf.extra_feature_dim > 0:
feats = torch.cat([feats, extra_feats], dim=1)
logits = clf.classifier_head(feats)
probs = torch.softmax(logits, dim=1)[:, 1].cpu().numpy().tolist()
for sid, prob, label in zip(ids, probs, labels):
writer.writerow([sid, prob, label])
print(f"Per-sample probabilities written to {args.output}")
if __name__ == "__main__":
main()
scripts/legacy/extract_best_auc.py
Executable
+115
View File
@@ -0,0 +1,115 @@
#!/usr/bin/env python3
"""
Quick utility to recover the best epoch metrics from HyperTower run folders.
Example:
python scripts/extract_best_auc.py analysis_data/img_only_densenet_gt_bin/img_only_densenet_gt_bin_20251028_112733
By default it looks for columns named like `auc_fused` (set via --metric) inside each
`fold{n}_epoch_log.csv`, returning the epoch with the highest value plus the holdout
metrics, if present.
"""
from __future__ import annotations
import argparse
import csv
import json
import math
from pathlib import Path
from typing import Dict, Optional, Tuple
def to_float(value: Optional[str]) -> Optional[float]:
if value is None:
return None
value = value.strip()
if not value:
return None
try:
out = float(value)
except ValueError:
return None
if math.isnan(out):
return None
return out
def best_row(path: Path, metric: str) -> Optional[Dict[str, str]]:
if not path.exists():
return None
best: Optional[Tuple[float, int, Dict[str, str]]] = None
with path.open("r", newline="") as fp:
reader = csv.DictReader(fp)
for row in reader:
val = to_float(row.get(metric))
if val is None:
continue
epoch = int(row.get("epoch", reader.line_num))
if best is None or val > best[0]:
best = (val, epoch, row)
return best[2] if best else None
def summarize_fold(row: Dict[str, str], metric: str) -> Dict[str, float]:
data: Dict[str, float] = {}
for key in (metric, f"holdout_{metric.split('_', 1)[-1]}", "holdout_auc_img", "holdout_auc_fused"):
val = to_float(row.get(key))
if val is not None:
data[key] = val
epoch_val = to_float(row.get("epoch"))
if epoch_val is not None:
data["epoch"] = int(epoch_val)
return data
def main() -> None:
ap = argparse.ArgumentParser(description="Extract best-per-fold metric from HyperTower runs.")
ap.add_argument("run_dir", type=Path, help="Run directory (contains fold*_epoch_log.csv)")
ap.add_argument("--metric", default="auc_fused", help="Metric column to maximise (default: auc_fused)")
ap.add_argument("--json", type=Path, default=None, help="Optional path to dump JSON summary")
args = ap.parse_args()
run_dir: Path = args.run_dir
metric: str = args.metric
if not run_dir.exists():
raise SystemExit(f"Run directory not found: {run_dir}")
fold_summaries: Dict[str, Dict[str, float]] = {}
metric_values = []
for csv_path in sorted(run_dir.glob("fold*_epoch_log.csv")):
best = best_row(csv_path, metric)
fold_name = csv_path.stem.replace("_epoch_log", "")
if best is None:
print(f"{fold_name}: no valid '{metric}' values found")
continue
summary = summarize_fold(best, metric)
fold_summaries[fold_name] = summary
val = summary.get(metric)
if val is not None:
metric_values.append(val)
holdout_val = summary.get(f"holdout_{metric.split('_', 1)[-1]}")
print(f"{fold_name}: epoch={summary.get('epoch')} {metric}={val:.4f}" if val is not None else f"{fold_name}: epoch={summary.get('epoch')}")
if holdout_val is not None:
print(f" holdout_{metric.split('_', 1)[-1]}={holdout_val:.4f}")
if metric_values:
mean_val = sum(metric_values) / len(metric_values)
print(f"\nMean best {metric}: {mean_val:.4f}")
if args.json:
payload = {
"run_dir": str(run_dir),
"metric": metric,
"folds": fold_summaries,
"mean_metric": (sum(metric_values) / len(metric_values)) if metric_values else None,
}
args.json.parent.mkdir(parents=True, exist_ok=True)
args.json.write_text(json.dumps(payload, indent=2))
print(f"Summary written to {args.json}")
if __name__ == "__main__":
main()
scripts/legacy/paper_metadata.py
Executable
+493
View File
@@ -0,0 +1,493 @@
import pandas as pd
from classes import HyperTower, ClinicalData, list_names, build_papila_clinical
from pathlib import Path
import shutil, json, textwrap
from datetime import datetime
import numpy as np
from typing import Dict, List, Tuple
from sklearn.model_selection import GroupKFold
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.metrics import roc_auc_score
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.preprocessing import label_binarize
def _proba_from_model(model, X):
if hasattr(model[-1], "predict_proba"):
return model.predict_proba(X)
dec = model.decision_function(X)
if dec.ndim == 1: # binary margins -> make 2-col
dec = np.stack([-dec, dec], axis=1)
e = np.exp(dec - dec.max(axis=1, keepdims=True))
return e / e.sum(axis=1, keepdims=True)
def _cv_auc_multiclass_per_class(X, y, groups, model, n_splits=5) -> np.ndarray:
"""
Returns a length-3 array of mean OvR AUCs for Class0/1/2 across GroupKFold.
Uses nan-safe means if a class is absent in a fold's test split.
"""
gkf = GroupKFold(n_splits=n_splits)
per_class_lists = [[], [], []]
for tr, te in gkf.split(X, y, groups):
model.fit(X[tr], y[tr])
proba = _proba_from_model(model, X[te])
y_te = y[te]
y_bin = label_binarize(y_te, classes=[0, 1, 2]) # (n,3)
for k in range(3):
yk = y_bin[:, k]
if yk.min() != yk.max(): # both classes present
per_class_lists[k].append(roc_auc_score(yk, proba[:, k]))
else:
per_class_lists[k].append(np.nan)
return np.array([np.nanmean(per_class_lists[k]) for k in range(3)], dtype=float)
def _cv_auc_binary(X, y, groups, model, n_splits=5) -> float:
mask = np.isin(y, [0, 1])
Xb, yb, gb = X[mask], y[mask], groups[mask]
gkf = GroupKFold(n_splits=n_splits)
aucs = []
for tr, te in gkf.split(Xb, yb, gb):
model.fit(Xb[tr], yb[tr])
if hasattr(model[-1], "predict_proba"):
p = model.predict_proba(Xb[te])[:, 1]
else:
p = model.decision_function(Xb[te])
# logistic squash for safety
if np.ptp(p) > 0:
p = 1.0 / (1.0 + np.exp(-p))
else:
p = np.full_like(p, 0.5, dtype=float)
# only compute if both classes present
if len(np.unique(yb[te])) == 2:
aucs.append(roc_auc_score(yb[te], p))
else:
aucs.append(np.nan)
return float(np.nanmean(aucs))
# -----------------------------------
# 1) Build Clinical Data (paper-faithful)
# -----------------------------------
IMAGE_DIR = "Papila/FundusImages"
CLINICAL_DIR = "Papila/ClinicalData"
LABEL_COL = "Diagnosis"
CAT_COLS = ["Gender", "Phakic/Pseudophakic"]
paper_auc = {
"TEST3_multiclass": { # Class0=Healthy, Class1=Glaucoma, Class2=Suspect
"LogReg": {"Class0": 0.67, "Class1": 0.66, "Class2": 0.67}, # from Fig. 7 (rounded)
"kNN": {"Class0": 0.72, "Class1": 0.70, "Class2": 0.76}, # your read of Fig. 7
"RF": {"Class0": 0.66, "Class1": 0.66, "Class2": 0.67}, # from Fig. 7 (rounded)
"SVM": {"Class0": 0.66, "Class1": 0.65, "Class2": 0.66}, # from Fig. 7 (rounded)
},
"TEST4_binary": { # Healthy vs Glaucoma (Suspects removed)
"LogReg": 0.71, # from text/Fig. 7 range midpoint
"kNN": 0.75, # your read of Fig. 7
"RF": 0.70, # from Fig. 7 (rounded)
"SVM": 0.69, # from Fig. 7 (rounded)
}
}
clinical = build_papila_clinical(
image_dir=IMAGE_DIR,
clinical_dir=CLINICAL_DIR,
label_col=LABEL_COL,
cat_cols=CAT_COLS,
)
# -----------------------------------
# 2) Feature matrix (no MD; IOP_corr already present)
# -----------------------------------
def build_feature_matrix(clinical) -> Tuple[np.ndarray, np.ndarray, np.ndarray, List[str]]:
"""
Returns:
X: features (N x D)
y: labels (Diagnosis: 0 healthy, 1 glaucoma, 2 suspect)
groups: patient IDs for GroupKFold
feat_names: list of feature names in X order
"""
df = clinical.df.copy()
# Scalars used in paper-style baselines (no VF_MD)
scalars = ["Age", "dioptre_1", "dioptre_2", "astigmatism",
"Pachymetry", "Axial_Length", "IOP_corr"]
# Categorical one-hot
cats = ["Gender", "Phakic/Pseudophakic"]
df_cats = pd.get_dummies(df[cats].astype("category"), drop_first=False, prefix=cats)
# Combine
X = pd.concat([df[scalars], df_cats], axis=1)
# Median impute numerics (simple, consistent)
for c in scalars:
med = pd.to_numeric(X[c], errors="coerce").median()
X[c] = pd.to_numeric(X[c], errors="coerce").fillna(med)
y = df[LABEL_COL].astype(int).values
groups = df["Patient ID"].astype(int).values
feat_names = list(X.columns)
return X.values.astype(np.float32), y, groups, feat_names
# ----------------------------
# 3) Model zoo (the four methods used in the paper)
# ----------------------------
def make_models(best_params: dict | None = None, random_state: int = 42) -> dict:
"""
Build paper-like baseline models. If best_params is provided (a dict mapping
model-name -> param dict with pipeline-style keys like 'clf__C'), those
params are applied to the corresponding pipelines.
"""
models = {
"LogReg": Pipeline([
("scaler", StandardScaler()),
("clf", LogisticRegression(
max_iter=100,
solver="lbfgs",
multi_class="auto"
))
]),
"kNN": Pipeline([
("scaler", StandardScaler()),
("clf", KNeighborsClassifier(
n_neighbors=5,
weights="uniform",
metric="minkowski",
p=2
))
]),
"RF": Pipeline([
("clf", RandomForestClassifier(
n_estimators=100,
criterion="gini",
max_depth=None,
min_samples_split=2,
min_samples_leaf=1,
max_features="sqrt",
bootstrap=True,
# random_state left as default; set via best_params if desired
))
]),
"SVM": Pipeline([
("scaler", StandardScaler()),
("clf", SVC(
C=1.0,
kernel="rbf",
gamma="scale",
probability=False
))
]),
}
# Apply overrides if provided
if best_params:
for name, params in best_params.items():
if name in models and params:
models[name].set_params(**params)
return models
# -----------------------------------
# 4) CV AUCs (mean over 5 folds; GroupKFold by patient)
# -----------------------------------
def _cv_auc_multiclass(X, y, groups, model, n_splits=5) -> float:
gkf = GroupKFold(n_splits=n_splits)
aucs = []
for tr, te in gkf.split(X, y, groups):
model.fit(X[tr], y[tr])
if hasattr(model[-1], "predict_proba"):
proba = model.predict_proba(X[te])
else:
dec = model.decision_function(X[te])
if dec.ndim == 1:
dec = np.stack([-dec, dec], axis=1)
e = np.exp(dec - dec.max(axis=1, keepdims=True))
proba = e / e.sum(axis=1, keepdims=True)
aucs.append(roc_auc_score(y[te], proba, multi_class="ovr", average="macro"))
return float(np.mean(aucs))
def _cv_auc_binary(X, y, groups, model, n_splits=5) -> float:
# Keep classes 0 (healthy) and 1 (glaucoma); drop suspects (2)
mask = np.isin(y, [0, 1])
Xb, yb, gb = X[mask], y[mask], groups[mask]
gkf = GroupKFold(n_splits=n_splits)
aucs = []
for tr, te in gkf.split(Xb, yb, gb):
model.fit(Xb[tr], yb[tr])
if hasattr(model[-1], "predict_proba"):
p = model.predict_proba(Xb[te])[:, 1]
else:
p = model.decision_function(Xb[te])
# simple logistic squashing if needed
if np.ptp(p) > 0:
p = 1.0 / (1.0 + np.exp(-p))
else:
p = np.full_like(p, 0.5, dtype=float)
aucs.append(roc_auc_score(yb[te], p))
return float(np.mean(aucs))
# -----------------------------------
# 5) Run both tests (multiclass + binary) and print table
# -----------------------------------
def run_papila_clinical_baselines(clinical, n_splits: int = 5,
random_state: int = 42,
best_params: dict | None = None) -> pd.DataFrame:
X, y, groups, feat_names = build_feature_matrix(clinical)
models = make_models(best_params=best_params, random_state=random_state)
rows = []
for name, model in models.items():
c0, c1, c2 = _cv_auc_multiclass_per_class(X, y, groups, model, n_splits=n_splits)
auc_bin = _cv_auc_binary(X, y, groups, model, n_splits=n_splits)
rows.append({"model": name, "Class0": c0, "Class1": c1, "Class2": c2, "Binary": auc_bin})
df = pd.DataFrame(rows).set_index("model").sort_index()
return df
results = run_papila_clinical_baselines(clinical, n_splits=5)
# print(results.to_string(float_format=lambda x: f"{x:.3f}"))
##############################
from sklearn.model_selection import ParameterGrid
from sklearn.base import clone
from sklearn.preprocessing import label_binarize
from sklearn.utils import check_random_state
# ==============================
# Helper: per-class & binary AUC with GroupKFold
# ==============================
def _proba_from_model(model, X):
if hasattr(model[-1], "predict_proba"):
return model.predict_proba(X)
# decision_function fallback
dec = model.decision_function(X)
if dec.ndim == 1: # binary margin -> 2-col probs
dec = np.stack([-dec, dec], axis=1)
e = np.exp(dec - dec.max(axis=1, keepdims=True))
return e / e.sum(axis=1, keepdims=True)
def _cv_auc_perclass_and_binary(X, y, groups, model, n_splits=5):
"""
Returns:
per_class_auc: length-3 array (Class0, Class1, Class2) averaged over folds
binary_auc: scalar (0 vs 1) averaged over folds
"""
gkf = GroupKFold(n_splits=n_splits)
# Hold fold-wise per-class AUCs (list of arrays of length 3)
perclass_fold_scores = []
binary_fold_scores = []
for tr, te in gkf.split(X, y, groups):
y_te = y[te]
# Multiclass per-class (OvR)
model.fit(X[tr], y[tr])
proba = _proba_from_model(model, X[te])
# One-vs-rest per-class AUCs (skip a class if absent in test fold)
y_bin = label_binarize(y_te, classes=[0, 1, 2]) # shape (n, 3)
perclass_scores = []
for k in range(3):
yk = y_bin[:, k]
# Only compute if both 0 and 1 are present
if yk.min() != yk.max():
perclass_scores.append(roc_auc_score(yk, proba[:, k]))
else:
perclass_scores.append(np.nan)
perclass_fold_scores.append(perclass_scores)
# Binary AUC (0 vs 1; drop class 2)
mask = np.isin(y_te, [0, 1])
if mask.sum() > 0 and len(np.unique(y_te[mask])) == 2:
# we need probabilities/margins for class 1 among (0,1)
# Map proba[:, 1] if the model was trained 3-way; we restrict te samples to 0/1
binary_p = proba[mask, 1]
binary_y = y_te[mask]
binary_fold_scores.append(roc_auc_score(binary_y, binary_p))
else:
binary_fold_scores.append(np.nan)
# Average over folds (ignore NaNs if a class was missing in a fold)
perclass_arr = np.array(perclass_fold_scores, dtype=float) # (n_folds, 3)
per_class_auc = np.nanmean(perclass_arr, axis=0)
binary_auc = float(np.nanmean(np.array(binary_fold_scores, dtype=float)))
return per_class_auc, binary_auc
# ==============================
# Distance-to-paper objective
# ==============================
def _distance_to_paper(model_name: str,
per_class_auc: np.ndarray,
binary_auc: float,
paper_auc: Dict,
w_mc: float = 1.0,
w_bin: float = 1.0) -> float:
mc_targets = paper_auc["TEST3_multiclass"][model_name]
tvec = np.array([mc_targets["Class0"], mc_targets["Class1"], mc_targets["Class2"]], dtype=float)
mc_diff = np.nanmean(np.abs(per_class_auc - tvec)) # mean absolute difference over 3 classes
bin_target = paper_auc["TEST4_binary"][model_name]
bin_diff = abs(binary_auc - bin_target)
return float(w_mc * mc_diff + w_bin * bin_diff)
# ==============================
# Parameter grids (paper-ish, not crazy-large)
# ==============================
def get_param_grids() -> Dict[str, List[dict]]:
return {
"LogReg": [
{
"clf__C": [0.01, 0.1, 1.0, 3.0, 10.0],
"clf__class_weight": [None, "balanced"],
"clf__max_iter": [200, 500],
# lbfgs + l2 is implied
}
],
"kNN": [
{
"clf__n_neighbors": [3, 5, 7, 9, 11],
"clf__weights": ["uniform", "distance"],
"clf__p": [1, 2], # Manhattan vs Euclidean
}
],
"RF": [
{
"clf__n_estimators": [200, 500, 1000],
"clf__max_depth": [None, 5, 10, 20],
"clf__max_features": ["sqrt", "log2", 0.5],
"clf__min_samples_leaf": [1, 2, 5],
"clf__class_weight": [None, "balanced"],
# If you want determinism add: "clf__random_state": [42]
}
],
"SVM": [
{
"clf__C": [0.1, 1.0, 3.0, 10.0],
"clf__gamma": ["scale", "auto", 0.1, 0.01, 0.001],
"clf__kernel": ["rbf"], # fixed to rbf as in paper-like default
}
],
}
# ==============================
# Grid search loop minimizing distance-to-paper
# ==============================
def search_params_to_match_paper(
clinical,
models: Dict[str, Pipeline],
paper_auc: Dict,
n_splits: int = 5,
w_mc: float = 1.0,
w_bin: float = 1.0,
verbose: bool = True,
) -> Tuple[pd.DataFrame, Dict[str, dict]]:
X, y, groups, feat_names = build_feature_matrix(clinical)
grids = get_param_grids()
summary_rows = []
best_params_by_model = {}
for name, base_model in models.items():
if name not in grids:
if verbose:
print(f"[warn] No grid for {name}, skipping.")
continue
best_loss = np.inf
best_params = None
best_mc = None
best_bin = None
for param_set in ParameterGrid(grids[name]):
model = clone(base_model).set_params(**param_set)
per_class_auc, binary_auc = _cv_auc_perclass_and_binary(
X, y, groups, model, n_splits=n_splits
)
loss = _distance_to_paper(
name, per_class_auc, binary_auc, paper_auc, w_mc=w_mc, w_bin=w_bin
)
if verbose:
mc_str = " / ".join(f"{a:.3f}" if np.isfinite(a) else "nan" for a in per_class_auc)
print(f"[{name}] params={param_set} | mc per-class={mc_str} | bin={binary_auc:.3f} | loss={loss:.4f}")
if loss < best_loss:
best_loss = loss
best_params = param_set
best_mc = per_class_auc
best_bin = binary_auc
# store
best_params_by_model[name] = best_params
summary_rows.append({
"model": name,
"best_loss": best_loss,
"best_params": json.dumps(best_params),
"mc_Class0": float(best_mc[0]),
"mc_Class1": float(best_mc[1]),
"mc_Class2": float(best_mc[2]),
"binary_auc": float(best_bin),
"paper_mc_Class0": paper_auc["TEST3_multiclass"][name]["Class0"],
"paper_mc_Class1": paper_auc["TEST3_multiclass"][name]["Class1"],
"paper_mc_Class2": paper_auc["TEST3_multiclass"][name]["Class2"],
"paper_binary": paper_auc["TEST4_binary"][name],
})
df = pd.DataFrame(summary_rows).set_index("model").sort_values("best_loss")
return df, best_params_by_model
# ==============================
# Run the search
# ==============================
models = make_models(random_state=42)
df_match, best_params = search_params_to_match_paper(
clinical=clinical,
models=models,
paper_auc=paper_auc,
n_splits=5,
w_mc=1.0, # weight multiclass distance
w_bin=1.0, # weight binary distance
verbose=True
)
# print("\n=== Best params found (by minimal distance-to-paper) ===")
# print(df_match[["best_loss","best_params","mc_Class0","mc_Class1","mc_Class2","binary_auc",
# "paper_mc_Class0","paper_mc_Class1","paper_mc_Class2","paper_binary"]])
# print("\nBest param dicts:")
for k, v in best_params.items():
print(k, "->", v)
results2 = run_papila_clinical_baselines(clinical, n_splits=5, random_state=42, best_params=best_params)
print(f"Default Settings: {results.round(2)}")
print(f"Best Params Settings: {results2.round(2)}")
print(f" Paper Results: {pd.DataFrame({
model: {**vals, "Binary": paper_auc["TEST4_binary"][model]}
for model, vals in paper_auc["TEST3_multiclass"].items()
}).T[["Class0","Class1","Class2","Binary"]]}")
scripts/legacy/run_all_sweep.sh
Executable
+119
View File
@@ -0,0 +1,119 @@
#!/usr/bin/env bash
set -euo pipefail
# Usage:
# bash scripts/run_all_sweep.sh --epochs 25 --n-splits 5 --batch-size 8 [extra args]
#
# Merged sweep: runs the SE attention grid (bridge/tower/both × R=8/16/32),
# skipping tower-only non-normalized variants (tower normalization is a no-op),
# and includes binary eval counterparts for each baseline run. It also submits
# the full gradual-thaw grid (multiclass + binary variants).
ARGS=("$@")
run() {
local SHORT="$1"; shift
echo "=== Running: $SHORT ==="
# Skip if a summary for this shortname already exists
if ls "analysis_data/${SHORT}_"*.md >/dev/null 2>&1; then
echo "… skipping ${SHORT} (summary already present)"
return 0
fi
python3 scripts/run_multifold.py \
--shortname "$SHORT" \
"$@" \
"${ARGS[@]}" || true
}
echo "--- SE Grid (bridge/tower/both × R=8/16/32; tower nonorm skipped) ---"
for R in 8 16 32; do
# Bridge-only
run "se_bridge_R${R}_norm" --se-where bridge --se-reduction ${R} --se-pre-norm --checkpoint-best
run "se_bridge_R${R}_norm_bin" --se-where bridge --se-reduction ${R} --se-pre-norm --checkpoint-best --eval_mode binary
run "se_bridge_R${R}_nonorm" --se-where bridge --se-reduction ${R} --no-se-pre-norm --checkpoint-best
run "se_bridge_R${R}_nonorm_bin" --se-where bridge --se-reduction ${R} --no-se-pre-norm --checkpoint-best --eval_mode binary
# Tower-only
run "se_tower_R${R}_norm" --se-where tower --se-reduction-tower ${R} --se-pre-norm-tower --checkpoint-best
run "se_tower_R${R}_norm_bin" --se-where tower --se-reduction-tower ${R} --se-pre-norm-tower --checkpoint-best --eval_mode binary
# Tower+Bridge
run "se_tower_bridge_R${R}_norm" \
--se-where both --se-reduction ${R} --se-reduction-tower ${R} \
--se-pre-norm --se-pre-norm-tower --checkpoint-best
run "se_tower_bridge_R${R}_norm_bin" \
--se-where both --se-reduction ${R} --se-reduction-tower ${R} \
--se-pre-norm --se-pre-norm-tower --checkpoint-best --eval_mode binary
run "se_tower_bridge_R${R}_nonorm" \
--se-where both --se-reduction ${R} --se-reduction-tower ${R} \
--no-se-pre-norm --no-se-pre-norm-tower --checkpoint-best
run "se_tower_bridge_R${R}_nonorm_bin" \
--se-where both --se-reduction ${R} --se-reduction-tower ${R} \
--no-se-pre-norm --no-se-pre-norm-tower --checkpoint-best --eval_mode binary
done
THAW_COMMON_ARGS=(
--gradual-thaw
--thaw-phase-duration 5
--thaw-ratio 0.33
--thaw-start-epoch 5
--early-stop
--early-patience 5
)
echo "--- Gradual Thaw Grid (multiclass + binary) ---"
# Bridge-only thaw runs (norm and nonorm)
for R in 8 16 32; do
for MODE in norm nonorm; do
if [[ "$MODE" == "norm" ]]; then
FLAGS=(--se-where bridge --se-reduction "$R" --se-pre-norm --checkpoint-best)
else
FLAGS=(--se-where bridge --se-reduction "$R" --no-se-pre-norm --checkpoint-best)
fi
run "thaw_se_bridge_R${R}_${MODE}" "${FLAGS[@]}" "${THAW_COMMON_ARGS[@]}"
run "thawbin_se_bridge_R${R}_${MODE}" "${FLAGS[@]}" "${THAW_COMMON_ARGS[@]}" --eval_mode binary
done
done
# Tower-only thaw runs (norm and nonorm)
for R in 8 16 32; do
for MODE in norm nonorm; do
if [[ "$MODE" == "norm" ]]; then
FLAGS=(--se-where tower --se-reduction-tower "$R" --se-pre-norm-tower --checkpoint-best)
else
FLAGS=(--se-where tower --se-reduction-tower "$R" --no-se-pre-norm-tower --checkpoint-best)
fi
run "thaw_se_tower_R${R}_${MODE}" "${FLAGS[@]}" "${THAW_COMMON_ARGS[@]}"
run "thawbin_se_tower_R${R}_${MODE}" "${FLAGS[@]}" "${THAW_COMMON_ARGS[@]}" --eval_mode binary
done
done
# Tower+bridge thaw runs (norm and nonorm)
for R in 8 16 32; do
for MODE in norm nonorm; do
if [[ "$MODE" == "norm" ]]; then
FLAGS=(
--se-where both
--se-reduction "$R"
--se-reduction-tower "$R"
--se-pre-norm
--se-pre-norm-tower
--checkpoint-best
)
else
FLAGS=(
--se-where both
--se-reduction "$R"
--se-reduction-tower "$R"
--no-se-pre-norm
--no-se-pre-norm-tower
--checkpoint-best
)
fi
run "thaw_se_tower_bridge_R${R}_${MODE}" "${FLAGS[@]}" "${THAW_COMMON_ARGS[@]}"
run "thawbin_se_tower_bridge_R${R}_${MODE}" "${FLAGS[@]}" "${THAW_COMMON_ARGS[@]}" --eval_mode binary
done
done
echo "Merged sweep submitted. Check analysis_data/* and models/* for outputs."
scripts/legacy/run_gradual_thaw_top5.sh
Executable
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env bash
set -euo pipefail
# Usage:
# bash scripts/run_gradual_thaw_top5.sh --epochs 20 --n-splits 5 --batch-size 8 [extra args]
#
# Runs the full gradual-thaw grid aligned with the SE sweep (bridge/tower/both × R=8/16/32 × norm vs nonorm).
ARGS=("$@")
run() {
local SHORT="$1"; shift
echo "=== Running: $SHORT ==="
if ls "analysis_data/${SHORT}_"*.md >/dev/null 2>&1; then
echo "… skipping ${SHORT} (summary already present)"
return 0
fi
python3 scripts/run_multifold.py \
--shortname "$SHORT" \
--gradual-thaw --thaw-phase-duration 5 --thaw-ratio 0.33 --thaw-start-epoch 5 \
--early-stop --early-patience 5 \
"$@" \
"${ARGS[@]}" || true
}
# Bridge-only thaw runs
for R in 8 16 32; do
for MODE in norm nonorm; do
if [[ "$MODE" == "norm" ]]; then
FLAGS=(--se-where bridge --se-reduction "$R" --se-pre-norm --checkpoint-best)
else
FLAGS=(--se-where bridge --se-reduction "$R" --no-se-pre-norm --checkpoint-best)
fi
run "thaw_se_bridge_R${R}_${MODE}" "${FLAGS[@]}"
run "thawbin_se_bridge_R${R}_${MODE}" "${FLAGS[@]}" --eval_mode binary
done
done
# Tower-only thaw runs
for R in 8 16 32; do
for MODE in norm nonorm; do
if [[ "$MODE" == "norm" ]]; then
FLAGS=(--se-where tower --se-reduction-tower "$R" --se-pre-norm-tower --checkpoint-best)
else
FLAGS=(--se-where tower --se-reduction-tower "$R" --no-se-pre-norm-tower --checkpoint-best)
fi
run "thaw_se_tower_R${R}_${MODE}" "${FLAGS[@]}"
run "thawbin_se_tower_R${R}_${MODE}" "${FLAGS[@]}" --eval_mode binary
done
done
# Tower+bridge thaw runs
for R in 8 16 32; do
for MODE in norm nonorm; do
if [[ "$MODE" == "norm" ]]; then
FLAGS=(
--se-where both
--se-reduction "$R"
--se-reduction-tower "$R"
--se-pre-norm
--se-pre-norm-tower
--checkpoint-best
)
else
FLAGS=(
--se-where both
--se-reduction "$R"
--se-reduction-tower "$R"
--no-se-pre-norm
--no-se-pre-norm-tower
--checkpoint-best
)
fi
run "thaw_se_tower_bridge_R${R}_${MODE}" "${FLAGS[@]}"
run "thawbin_se_tower_bridge_R${R}_${MODE}" "${FLAGS[@]}" --eval_mode binary
done
done
echo "Gradual thaw grid submitted. Check analysis_data/* and models/* for outputs."
scripts/legacy/run_multifold_gui.py
Executable
+15
View File
@@ -0,0 +1,15 @@
#!/usr/bin/env python3
"""Launch the Tkinter front-end for run_multifold."""
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.frontend import launch_frontend
if __name__ == "__main__":
launch_frontend()
scripts/legacy/run_multimodel.py
Executable
+145
View File
@@ -0,0 +1,145 @@
#!/usr/bin/env python3
import argparse, subprocess, sys, time, json
from pathlib import Path
# Backbones in the paper that torchvision supports
BACKBONES = [
"efficientnet_b0",
"resnet50",
"densenet121",
"vgg16",
"mobilenet_v2",
"inception_v3",
# (Xception omitted; not in torchvision — add via timm later if needed)
]
MODES = [
("multiclass", ["Healthy", "Glaucoma", "Suspect"]),
("binary", ["Healthy", "Glaucoma"]),
]
def run(cmd):
print("\n$ " + " ".join(map(str, cmd)))
res = subprocess.run(cmd, check=True)
return res.returncode
def main():
ap = argparse.ArgumentParser(description="Run all paper CNNs across folds in multiclass + binary, then compile plots.")
ap.add_argument("--epochs", type=int, default=5, help="Epochs per fold (fast sanity first).")
ap.add_argument("--shortname", type=str, default="papergrid", help="Prefix for run IDs.")
ap.add_argument("--n-splits", type=int, default=5, help="Number of folds.")
ap.add_argument("--fusion-mode", type=str, default="fused", choices=["image_only","fused","metadata_only","vote"],
help="Paper CNNs are image-only; leave as image_only unless you’re testing others.")
ap.add_argument("--freeze-ratio", type=float, default=0.0, help="0.0 = full fine-tune (as in the paper).")
# You can override data roots if needed
ap.add_argument("--image-dir", default="Papila/FundusImages")
ap.add_argument("--clinical-dir", default="Papila/ClinicalData")
ap.add_argument("--label-col", default="Diagnosis")
ap.add_argument("--cat-cols", nargs="*", default=["Gender", "Phakic/Pseudophakic"])
args = ap.parse_args()
ts = time.strftime("%Y%m%d_%H%M%S")
master_tag = f"{args.shortname}_{ts}"
master_dir = Path("analysis_data") / master_tag
master_dir.mkdir(parents=True, exist_ok=True)
# Keep a log of all subruns for the master report
index = []
for backbone in BACKBONES:
for eval_mode, class_names in MODES:
# build a child shortname per (backbone, mode)
sub_prefix = f"{args.shortname}_{backbone}_{eval_mode}"
cmd = [
sys.executable, "scripts/run_multifold.py",
"--backbone", backbone,
"--freeze-ratio", str(args.freeze_ratio),
"--fusion-mode", args.fusion_mode,
"--epochs", str(args.epochs),
"--n-splits", str(args.n_splits),
"--shortname", sub_prefix,
"--eval_mode", eval_mode,
"--image-dir", args.image_dir,
"--clinical-dir", args.clinical_dir,
"--label-col", args.label_col,
]
# class names by mode (ensures plot legends are correct)
cmd += ["--class-names", *class_names]
plot_head_map = {
"image_only": "image",
"fused" : "fused",
"metadata_only": "metadata",
"vote": "fused",
}
# We always aggregate/plot the image head for paper CNNs
cmd += ["--plot-head", plot_head_map.get(args.fusion_mode)]
# Delegate the whole run to run_multifold.py
run(cmd)
# Discover the child run folder (the newest folder matching the shortname prefix)
# We do this because run_multifold appends its own timestamp.
adir = Path("analysis_data")
children = sorted([p for p in adir.glob(f"{sub_prefix}_*") if p.is_dir()])
if not children:
print(f"[WARN] No analysis_data folder found for {sub_prefix}; skipping index entry.")
continue
run_dir = children[-1]
summary_json = run_dir / "summary.json"
plots_dir = run_dir / "plots"
# Record entry
entry = {
"backbone": backbone,
"eval_mode": eval_mode,
"run_dir": str(run_dir),
"summary_json": str(summary_json) if summary_json.exists() else None,
"plots": {
"mean": str(plots_dir / "roc_image_mean_ovr.png"),
"overlay": str(plots_dir / "roc_image_perfold_overlay.png"),
}
}
# Try to read AUCs
try:
if summary_json.exists():
entry.update(json.loads(summary_json.read_text()))
except Exception:
pass
index.append(entry)
# Write a master JSON + markdown report
(master_dir / "index.json").write_text(json.dumps(index, indent=2), encoding="utf-8")
# Simple markdown table of results with links
lines = [
f"# Multimodel grid — {master_tag}",
"",
f"- Epochs per fold: **{args.epochs}**",
f"- Folds: **{args.n_splits}**",
f"- Fusion mode: **{args.fusion_mode}** (paper CNNs = image-only)",
f"- Freeze ratio: **{args.freeze_ratio}**",
"",
"| Backbone | Mode | Mean AUC (macro/mc or ROC-AUC/bin) | Plots | Run folder |",
"|---|---|---:|---|---|",
]
for e in index:
auc_mean = e.get("macro_ovr_auc_mean", None)
if auc_mean is not None:
auc_str = f"{auc_mean:.3f}"
else:
auc_str = "—"
mean_png = e["plots"]["mean"]
overlay_png = e["plots"]["overlay"]
plots_md = f"[mean]({mean_png}) / [overlay]({overlay_png})"
lines.append(
f"| `{e['backbone']}` | `{e['eval_mode']}` | {auc_str} | {plots_md} | `{e['run_dir']}` |"
)
(master_dir / "README.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
print(f"\nAll done.\n- Master index: {master_dir/'index.json'}\n- Report: {master_dir/'README.md'}")
print(f"- Individual runs live under analysis_data/<shortname_backbone_mode_*> with plots and summaries.")
if __name__ == "__main__":
main()
scripts/legacy/run_se_sweep.sh
Executable
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env bash
set -euo pipefail
# Usage:
# bash scripts/run_se_sweep.sh --epochs 50 --n-splits 5 --batch-size 8 --eval_mode multiclass [extra args]
#
# This will launch a series of runs covering the grid from the slide:
# - Bridge-only R8/R16/R32 (normalized and non-normalized)
# - Tower-only R8/R16/R32 (normalized and non-normalized)
# - Tower+Bridge R8/R16/R32 (normalized and non-normalized)
ARGS=("$@")
run() {
local SHORT="$1"; shift
echo "=== Running: $SHORT ==="
# Skip if a summary for this shortname already exists
if ls "analysis_data/${SHORT}_"*.md >/dev/null 2>&1; then
echo "… skipping ${SHORT} (summary already present)"
return 0
fi
python3 scripts/run_multifold.py \
--shortname "$SHORT" \
"$@" \
"${ARGS[@]}" || true
}
# Bridge-only (normalized + non-normalized)
for R in 8 16 32; do
run "se_bridge_R${R}_norm" --se-where bridge --se-reduction ${R} --se-pre-norm --checkpoint-best
run "se_bridge_R${R}_nonorm" --se-where bridge --se-reduction ${R} --no-se-pre-norm --checkpoint-best
done
# Tower-only (normalized + non-normalized)
for R in 8 16 32; do
run "se_tower_R${R}_norm" \
--se-where tower --se-reduction-tower ${R} --se-pre-norm-tower \
--checkpoint-best
run "se_tower_R${R}_nonorm" \
--se-where tower --se-reduction-tower ${R} --no-se-pre-norm-tower \
--checkpoint-best
done
# Tower+Bridge (normalized + non-normalized)
for R in 8 16 32; do
# normalized (both pre-norm on)
run "se_tower_bridge_R${R}_norm" \
--se-where both --se-reduction ${R} --se-reduction-tower ${R} \
--se-pre-norm --se-pre-norm-tower --checkpoint-best
# non-normalized (both pre-norm off)
run "se_tower_bridge_R${R}_nonorm" \
--se-where both --se-reduction ${R} --se-reduction-tower ${R} \
--no-se-pre-norm --no-se-pre-norm-tower --checkpoint-best
done
echo "Sweep submitted. Check analysis_data/* and models/* for outputs."
scripts/main/pipeline.ipynb
+178
View File
@@ -0,0 +1,178 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Hypertower Repro Pipeline\n",
"\n",
"This notebook documents the full run sequence used to reproduce current results."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 0) Environment + Paths\n",
"\n",
"- Activate `fundus_imaging` environment\n",
"- Run from repo root\n",
"- Confirm data paths:\n",
" - `Papila/FundusImages`\n",
" - `Papila/ClinicalData`\n",
" - `Papila/ExpertsSegmentations`"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pathlib import Path\n",
"\n",
"required = [\n",
" Path(\"Papila/FundusImages\"),\n",
" Path(\"Papila/ClinicalData\"),\n",
" Path(\"Papila/ExpertsSegmentations\"),\n",
" Path(\"REFUGE\"),\n",
"]\n",
"for p in required:\n",
" print(f\"{p}:\", \"OK\" if p.exists() else \"MISSING\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 1) Build UNet Manifest"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!python3 scripts/main/refuge/build_manifest.py --output manifest.csv"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 2) Train UNet Segmenter (per-image normalization)\n",
"\n",
"Current tuned baseline:\n",
"- `--device cuda`\n",
"- `--batch-size 8`\n",
"- `--loader-workers 14`\n",
"- `--in-memory-cache`\n",
"- `--cache-workers 4`"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!python3 scripts/run_unet_segmenter.py \\\n",
" --manifest manifest.csv \\\n",
" --train --evaluate \\\n",
" --normalize per_image \\\n",
" --train-datasets refuge --val-datasets refuge --holdout-datasets refuge \\\n",
" --epochs 40 --batch-size 8 \\\n",
" --device cuda --loader-workers 14 \\\n",
" --in-memory-cache --cache-workers 4 \\\n",
" --checkpoint-dir models/v2/refuge/segmentation/per_image \\\n",
" --eval-output analysis_data/segmenter_eval/v2_refuge_per_image \\\n",
" --eval-metrics-path analysis_data/segmenter_eval/v2_refuge_per_image/metrics.csv"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 3) Run V2 Hypertower Modes (cropped with UNet)\n",
"\n",
"Runs binary + multiclass across:\n",
"- `single`\n",
"- `ensemble`\n",
"- `bilateral`"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!python3 scripts/basic_analysis/compare_hypertower_modes.py \\\n",
" --eval-modes binary multiclass \\\n",
" --tower-modes single ensemble bilateral \\\n",
" --epochs 40 \\\n",
" --n-splits 5 \\\n",
" --batch-size 8 \\\n",
" --backbone refugelike \\\n",
" --img-crop-manifest manifest.csv \\\n",
" --img-crop-weights models/v2/refuge/segmentation/per_image/best.pt \\\n",
" --img-crop-normalize per_image \\\n",
" --img-crop-cache analysis_data/v2_crops_unet_refuge \\\n",
" --run-name v2_modes_full_40ep_5fold_unet_perimage"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 4) Quick Result Snapshot"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"from pathlib import Path\n",
"\n",
"root = Path(\"analysis_data/v2_modes_full_40ep_5fold_unet_perimage\")\n",
"summary = root / \"summary.json\"\n",
"if summary.exists():\n",
" data = json.loads(summary.read_text())\n",
" print(\"run_name:\", data.get(\"run_name\"))\n",
" print(\"timestamp:\", data.get(\"timestamp\"))\n",
" print(\"keys:\", list(data.get(\"summaries\", {}).keys()))\n",
"else:\n",
" print(\"Summary not found:\", summary)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 5) Notes / Decisions\n",
"\n",
"- Mixed-label patient handling used:\n",
"- Warmup settings used:\n",
"- Backbone / batch / workers used:\n",
"- Any deviations from default run:"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"name": "python",
"version": "3.12"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
scripts/main/refuge/build_manifest.py
Executable
+140
View File
@@ -0,0 +1,140 @@
"""Build manifest for U-Net segmenter combining REFUGE and Papila annotations."""
from __future__ import annotations
import argparse
import random
from pathlib import Path
from typing import Optional
import pandas as pd
import sys
REPO_ROOT = Path(__file__).resolve().parents[3]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.refuge_preprocessing import RefugePreprocessing
REFUGE_ROOT = Path("REFUGE")
PAPILA_IMAGES = Path("Papila/FundusImages")
PAPILA_CONTOURS = Path("Papila/ExpertsSegmentations/Contours")
DEFAULT_OUTPUT = Path("manifest.csv")
def pick_contour(base: str, kind: str) -> Optional[Path]:
"""Return contour path for Papila image (disc/cup)."""
candidates = [
PAPILA_CONTOURS / f"{base}_{kind}_exp2.txt",
PAPILA_CONTOURS / f"{base}_{kind}_exp1.txt",
]
for path in candidates:
if path.exists():
return path
return None
def collect_refuge() -> pd.DataFrame:
pre = RefugePreprocessing(REFUGE_ROOT)
samples = []
for sample in pre.build_manifest(refresh=True):
if sample.mask_path is None:
continue
split = sample.split
if split == "test":
split = "holdout"
samples.append(
{
"sample_id": sample.sample_id,
"dataset": "refuge",
"image_path": sample.image_path.resolve(),
"annotation_disc": sample.mask_path.resolve(),
"annotation_cup": sample.mask_path.resolve(),
"annotation_type_disc": "mask",
"annotation_type_cup": "mask",
"split": split,
}
)
return pd.DataFrame(samples)
def collect_papila() -> pd.DataFrame:
samples = []
if not PAPILA_IMAGES.exists():
return pd.DataFrame(samples)
for img_path in sorted(PAPILA_IMAGES.glob("RET*")):
base = img_path.stem
disc = pick_contour(base, "disc")
cup = pick_contour(base, "cup")
if disc is None or cup is None:
continue
samples.append(
{
"sample_id": f"papila_{base}",
"dataset": "papila",
"image_path": img_path.resolve(),
"annotation_disc": disc.resolve(),
"annotation_cup": cup.resolve(),
"annotation_type_disc": "contour",
"annotation_type_cup": "contour",
}
)
return pd.DataFrame(samples)
def assign_splits(df: pd.DataFrame, holdout_ratio: float, seed: int) -> pd.DataFrame:
rng = random.Random(seed)
df = df.copy()
if "split" not in df.columns:
df["split"] = None
for dataset, group in df.groupby("dataset"):
indices = list(group.index)
# Preserve provided splits (e.g., REFUGE train/val/test); only populate
# missing entries with "train" so downstream code has a default.
split_series = df.loc[indices, "split"]
missing = split_series.isna() | (split_series.astype(str).str.strip() == "")
if missing.any():
df.loc[missing[missing].index, "split"] = "train"
split_series = df.loc[indices, "split"]
if dataset != "papila":
continue
if holdout_ratio <= 0:
continue
desired_holdout = max(1, int(len(indices) * holdout_ratio))
split_series = df.loc[indices, "split"]
current_holdout_mask = split_series == "holdout"
current_holdout = int(current_holdout_mask.sum())
remaining = desired_holdout - current_holdout
if remaining <= 0:
continue
candidate_indices = list(split_series[split_series == "train"].index)
rng.shuffle(candidate_indices)
selected = candidate_indices[:remaining]
df.loc[selected, "split"] = "holdout"
return df
def main() -> None:
parser = argparse.ArgumentParser(description="Build U-Net manifest")
parser.add_argument("--holdout", type=float, default=0.05)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
args = parser.parse_args()
refuge_df = collect_refuge()
papila_df = collect_papila()
combined = pd.concat([refuge_df, papila_df], ignore_index=True)
combined = assign_splits(combined, holdout_ratio=args.holdout, seed=args.seed)
args.output.parent.mkdir(parents=True, exist_ok=True)
combined.to_csv(args.output, index=False)
print(f"Manifest saved to {args.output} with {len(combined)} entries")
if __name__ == "__main__":
main()
scripts/main/refuge/build_unet_manifest.py
+138
View File
@@ -0,0 +1,138 @@
"""Build manifest for U-Net segmenter combining REFUGE and Papila annotations."""
from __future__ import annotations
import argparse
import random
from pathlib import Path
from typing import Optional
import sys
import pandas as pd
ROOT = Path(__file__).resolve().parents[1]
sys.path.append(str(ROOT))
from classes.refuge_preprocessing import RefugePreprocessing
REFUGE_ROOT = Path("REFUGE")
PAPILA_IMAGES = Path("FundusImages")
PAPILA_CONTOURS = Path("Papila/ExpertsSegmentations/Contours")
DEFAULT_OUTPUT = Path("Papila/analysis_data/unet_manifest.csv")
def pick_contour(base: str, kind: str) -> Optional[Path]:
"""Return contour path for Papila image (disc/cup)."""
candidates = [
PAPILA_CONTOURS / f"{base}_{kind}_exp2.txt",
PAPILA_CONTOURS / f"{base}_{kind}_exp1.txt",
]
for path in candidates:
if path.exists():
return path
return None
def collect_refuge() -> pd.DataFrame:
pre = RefugePreprocessing(REFUGE_ROOT)
samples = []
for sample in pre.build_manifest(refresh=True):
if sample.mask_path is None:
continue
split = sample.split
if split == "test":
split = "holdout"
samples.append(
{
"sample_id": sample.sample_id,
"dataset": "refuge",
"image_path": sample.image_path.resolve(),
"annotation_disc": sample.mask_path.resolve(),
"annotation_cup": sample.mask_path.resolve(),
"annotation_type_disc": "mask",
"annotation_type_cup": "mask",
"split": split,
}
)
return pd.DataFrame(samples)
def collect_papila() -> pd.DataFrame:
samples = []
if not PAPILA_IMAGES.exists():
return pd.DataFrame(samples)
for img_path in sorted(PAPILA_IMAGES.glob("RET*")):
base = img_path.stem
disc = pick_contour(base, "disc")
cup = pick_contour(base, "cup")
if disc is None or cup is None:
continue
samples.append(
{
"sample_id": f"papila_{base}",
"dataset": "papila",
"image_path": img_path.resolve(),
"annotation_disc": disc.resolve(),
"annotation_cup": cup.resolve(),
"annotation_type_disc": "contour",
"annotation_type_cup": "contour",
}
)
return pd.DataFrame(samples)
def assign_splits(df: pd.DataFrame, holdout_ratio: float, seed: int) -> pd.DataFrame:
rng = random.Random(seed)
df = df.copy()
if "split" not in df.columns:
df["split"] = None
for dataset, group in df.groupby("dataset"):
indices = list(group.index)
# Preserve provided splits (e.g., REFUGE train/val/test); only populate
# missing entries with "train" so downstream code has a default.
split_series = df.loc[indices, "split"]
missing = split_series.isna() | (split_series.astype(str).str.strip() == "")
if missing.any():
df.loc[missing[missing].index, "split"] = "train"
split_series = df.loc[indices, "split"]
if dataset != "papila":
continue
if holdout_ratio <= 0:
continue
desired_holdout = max(1, int(len(indices) * holdout_ratio))
split_series = df.loc[indices, "split"]
current_holdout_mask = split_series == "holdout"
current_holdout = int(current_holdout_mask.sum())
remaining = desired_holdout - current_holdout
if remaining <= 0:
continue
candidate_indices = list(split_series[split_series == "train"].index)
rng.shuffle(candidate_indices)
selected = candidate_indices[:remaining]
df.loc[selected, "split"] = "holdout"
return df
def main() -> None:
parser = argparse.ArgumentParser(description="Build U-Net manifest")
parser.add_argument("--holdout", type=float, default=0.05)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
args = parser.parse_args()
refuge_df = collect_refuge()
papila_df = collect_papila()
combined = pd.concat([refuge_df, papila_df], ignore_index=True)
combined = assign_splits(combined, holdout_ratio=args.holdout, seed=args.seed)
combined.to_csv(args.output, index=False)
print(f"Manifest saved to {args.output} with {len(combined)} entries")
if __name__ == "__main__":
main()
scripts/main/refuge/refuge_build.py
+976
View File
@@ -0,0 +1,976 @@
"""REFUGE training/evaluation helper.
Usage examples (after activating .venv_refuge):
python refuge_build.py --train-seg
python refuge_build.py --train-clf
python refuge_build.py --eval --with-ttt
The script expects the REFUGE folder and writes checkpoints under
models/refuge/segmentation and models/refuge/classifier.
"""
from __future__ import annotations
import argparse
import csv
from pathlib import Path
from typing import Dict, List, Optional, Sequence, Set, Tuple
import shutil
import sys
import torch
import numpy as np
from PIL import Image, ImageDraw
from torch.utils.data import DataLoader
from sklearn.metrics import roc_auc_score
from tqdm import tqdm
from torch import nn
from torchvision import models
REPO_ROOT = Path(__file__).resolve().parents[3]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.refuge_preprocessing import RefugePreprocessing, RefugeSample
from classes.refuge_segmentation import RefugeSegmentation
from classes.refuge_classification import (
RefugeClassification,
RefugeClassificationRecord,
RefugeClassificationDataset,
_default_image_transform,
_geometry_from_mask,
UNetGeometryProvider,
)
from classes.unet_segmenter import UNetSegmenter
from classes.papila_builders import build_papila_clinical
REFUGE_ROOT = Path("REFUGE")
SEG_CKPT = Path("models/refuge/segmentation/refuge_segmentation_best.pt")
CLF_DIR = Path("models/refuge/classifier")
UNET_WEIGHT_CANDIDATES = (
Path("models/v2/refuge/segmentation/per_image/best.pt"),
Path("models/v2/refuge/segmentation/best.pt"),
Path("models/unet_segmenter/best.pt"),
)
CLASSIFIER_BACKBONES = {
"resnet50": models.ResNet50_Weights.DEFAULT,
"densenet121": models.DenseNet121_Weights.DEFAULT,
"efficientnet_b0": models.EfficientNet_B0_Weights.DEFAULT,
"efficientnet_b7": models.EfficientNet_B7_Weights.DEFAULT,
}
def build_classifier_backbone(name: str) -> nn.Module:
name = name.lower()
if name not in CLASSIFIER_BACKBONES:
raise ValueError(f"Unsupported classifier backbone '{name}'")
weights = CLASSIFIER_BACKBONES[name]
if name == "resnet50":
model = models.resnet50(weights=weights)
feat_dim = model.fc.in_features
model.fc = nn.Identity()
elif name == "densenet121":
model = models.densenet121(weights=weights)
feat_dim = model.classifier.in_features
model.classifier = nn.Identity()
elif name == "efficientnet_b0":
model = models.efficientnet_b0(weights=weights)
feat_dim = model.classifier[-1].in_features # type: ignore[index]
model.classifier = nn.Identity()
elif name == "efficientnet_b7":
model = models.efficientnet_b7(weights=weights)
feat_dim = model.classifier[-1].in_features # type: ignore[index]
model.classifier = nn.Identity()
else: # pragma: no cover
raise ValueError(f"Unsupported classifier backbone '{name}'")
setattr(model, "_feature_dim", int(feat_dim))
return model
def classifier_checkpoint_dir(backbone_name: str) -> Path:
return CLF_DIR / backbone_name
def classifier_checkpoint_path(backbone_name: str) -> Path:
return classifier_checkpoint_dir(backbone_name) / "refuge_classifier_best.pt"
def resolve_unet_weights(explicit: Optional[Path]) -> Path:
if explicit is not None:
return explicit
for cand in UNET_WEIGHT_CANDIDATES:
if cand.exists():
return cand
return UNET_WEIGHT_CANDIDATES[0]
def ensure_preprocessing() -> RefugePreprocessing:
if not REFUGE_ROOT.exists():
raise FileNotFoundError(f"REFUGE directory not found at {REFUGE_ROOT}")
return RefugePreprocessing(REFUGE_ROOT)
def load_allowed_ids(
csv_path: Optional[Path], dice_threshold: float
) -> Optional[Set[str]]:
if csv_path is None or not csv_path.exists():
return None
allowed: Set[str] = set()
with csv_path.open(newline="") as fh:
reader = csv.DictReader(fh)
for row in reader:
sample_id = row.get("sample_id")
if not sample_id or sample_id == "__mean__":
continue
try:
disc = float(row.get("dice_disc", "nan"))
cup = float(row.get("dice_cup", "nan"))
except (TypeError, ValueError):
continue
if disc < dice_threshold and cup < dice_threshold:
continue
allowed.add(sample_id)
return allowed
def build_papila_samples(
image_dir: Path,
clinical_dir: Path,
label_col: str,
positive_labels: Sequence[str],
allowed_ids: Optional[Set[str]],
) -> List[RefugeSample]:
clinical = build_papila_clinical(
image_dir=str(image_dir),
clinical_dir=str(clinical_dir),
label_col=label_col,
cat_cols=[],
)
positives = {lbl.lower() for lbl in positive_labels}
samples: Dict[str, RefugeSample] = {}
for _, row in clinical.df.iterrows():
image_path = clinical.get_image_path(row)
sample_id = f"papila_{Path(image_path).stem}"
if allowed_ids is not None and sample_id not in allowed_ids:
continue
if sample_id in samples:
continue
value = row.get(label_col)
if value is None or (isinstance(value, float) and np.isnan(value)):
continue
try:
label_int = int(value)
if label_int == 2:
continue
label = 1 if label_int > 0 else 0
except (TypeError, ValueError):
label = 1 if str(value).strip().lower() in positives else 0
samples[sample_id] = RefugeSample(
sample_id=sample_id,
dataset="papila",
split="holdout",
image_path=Path(image_path),
label=label,
device=None,
mask_path=None,
fovea_coord=None,
)
return list(samples.values())
def load_contour(path: Path) -> np.ndarray:
coords = np.loadtxt(path)
if coords.ndim == 1:
coords = coords.reshape(-1, 2)
return coords
def contour_to_mask(coords: np.ndarray, size: Tuple[int, int]) -> np.ndarray:
if coords is None or coords.size == 0:
return np.zeros((size[1], size[0]), dtype=np.uint8)
img = Image.new("L", size, 0)
draw = ImageDraw.Draw(img)
points = [tuple(map(float, pt)) for pt in coords]
draw.polygon(points, outline=1, fill=1)
return np.array(img, dtype=np.uint8)
class PapilaGTGeometryProvider:
def __init__(self, contours_dir: Path) -> None:
self.contours_dir = contours_dir
def _pick(self, base: str, kind: str) -> Optional[Path]:
for exp in ("exp2", "exp1"):
cand = self.contours_dir / f"{base}_{kind}_{exp}.txt"
if cand.exists():
return cand
return None
def __call__(self, sample: RefugeSample, scale: float):
base = Path(sample.image_path).stem
disc_path = self._pick(base, "disc")
cup_path = self._pick(base, "cup")
if disc_path is None or cup_path is None:
raise RuntimeError(f"Missing ground-truth contours for {sample.sample_id}")
image = Image.open(sample.image_path).convert("RGB")
disc_coords = load_contour(disc_path)
cup_coords = load_contour(cup_path)
disc_mask = contour_to_mask(disc_coords, image.size)
cup_mask = contour_to_mask(cup_coords, image.size)
cup_mask = ((cup_mask > 0) & (disc_mask > 0)).astype(np.uint8)
geom = _geometry_from_mask(disc_mask, scale)
return geom, disc_mask.astype(np.uint8), cup_mask.astype(np.uint8)
def build_papila_records(
args: argparse.Namespace,
pre: RefugePreprocessing,
checkpoint_path: Path,
) -> Tuple[List[RefugeClassificationRecord], Optional[RefugeClassification]]:
allowed = load_allowed_ids(
getattr(args, "papila_metrics", None),
getattr(args, "papila_dice_threshold", 0.01),
)
samples = build_papila_samples(
args.papila_image_dir,
args.papila_clinical_dir,
args.papila_label_col,
args.papila_positive_labels,
allowed,
)
if not samples:
return [], None
cache_dir = args.clf_cache_dir
if cache_dir is not None and getattr(args, "papila_use_gt", False):
cache_dir = cache_dir / "gt"
if getattr(args, "papila_use_gt", False):
geometry_fn = PapilaGTGeometryProvider(args.papila_contours_dir)
provider = geometry_fn
else:
seg_manifest = getattr(args, "seg_manifest", None)
seg_weights = resolve_unet_weights(getattr(args, "seg_weights", None))
if seg_manifest is None or seg_weights is None:
raise SystemExit(
"Papila evaluation without GT masks requires --seg-manifest and --seg-weights"
)
segmenter = UNetSegmenter(
manifest_path=seg_manifest,
device=args.device,
normalize=args.seg_normalize,
)
seg_state = torch.load(seg_weights, map_location=args.device)
seg_state_dict = seg_state.get("model", seg_state)
segmenter.model.load_state_dict(seg_state_dict)
segmenter.model.to(args.device)
provider = UNetGeometryProvider(
segmenter=segmenter,
threshold=args.segmenter_threshold,
tta=args.segmenter_tta,
)
geometry_fn = provider
papila_seg = RefugeSegmentation(pre)
backbone = build_classifier_backbone(args.clf_backbone)
papila_clf = RefugeClassification(
pre,
papila_seg,
backbone=backbone,
geometry_fn=provider,
cache_dir=cache_dir,
)
papila_clf.crop_scale = args.crop_scale
papila_clf.crop_size = args.crop_size
papila_clf.eval_transform = _default_image_transform(args.crop_size)
papila_clf.ttt_transform = papila_clf.eval_transform
papila_state = torch.load(checkpoint_path, map_location=args.device)
papila_clf.backbone.load_state_dict(papila_state["backbone"])
papila_clf.classifier_head.load_state_dict(papila_state["classifier"])
papila_clf.rotation_head.load_state_dict(papila_state["rotation"])
papila_clf.backbone.to(args.device)
papila_clf.classifier_head.to(args.device)
papila_clf.rotation_head.to(args.device)
records = papila_clf.build_records_for_samples(
samples, crop_scale=args.crop_scale, progress_prefix="papila"
)
print(f"[eval] Prepared {len(records)} PAPILA records")
return records, papila_clf
def train_segmentation(args: argparse.Namespace) -> None:
pre = ensure_preprocessing()
seg = RefugeSegmentation(pre)
seg.build_datasets(
image_size=args.seg_image_size,
batch_size=args.seg_batch_size,
num_workers=args.num_workers,
)
history = seg.train(
epochs=args.seg_epochs,
lr=args.seg_lr,
weight_decay=args.seg_weight_decay,
checkpoint_dir=SEG_CKPT.parent,
device=args.device,
)
print("Segmentation training complete. Best Dice:", history.get("best_dice"))
def train_unet_segmenter(args: argparse.Namespace) -> None:
manifest_path = args.seg_manifest or Path("manifest.csv")
mask_cache_dir = None if args.in_memory_cache else args.mask_cache_dir
image_cache_dir = None if args.in_memory_cache else args.image_cache_dir
if args.in_memory_cache and (args.mask_cache_dir or args.image_cache_dir):
print("[unet-seg] in_memory_cache enabled: disk caches disabled for this run.")
segmenter = UNetSegmenter(
manifest_path=manifest_path,
device=args.device,
target_size=args.seg_image_size,
normalize=args.seg_normalize,
use_stronger_aug=args.seg_strong_aug,
train_datasets=args.seg_train_datasets,
val_datasets=args.seg_val_datasets,
holdout_datasets=args.seg_holdout_datasets,
mask_cache_dir=mask_cache_dir,
image_cache_dir=image_cache_dir,
in_memory_cache=args.in_memory_cache,
loader_workers=args.loader_workers,
)
if mask_cache_dir:
print(f"[unet-seg] mask_cache_dir={mask_cache_dir}")
if image_cache_dir:
print(f"[unet-seg] image_cache_dir={image_cache_dir}")
if args.in_memory_cache:
print("[unet-seg] prebuilding in-memory cache")
segmenter.prebuild_in_memory_cache(
cache_workers=max(0, int(args.cache_workers)),
include_train=True,
include_val=True,
include_holdout=False,
)
segmenter.train(
epochs=args.seg_epochs,
batch_size=args.seg_batch_size,
lr=args.seg_lr,
weight_decay=args.seg_weight_decay,
checkpoint_dir=args.seg_checkpoint_dir,
)
print(
"[unet-seg] Training complete. Best checkpoint stored at",
(args.seg_checkpoint_dir / "best.pt").resolve(),
)
def _load_segmentation(
pre: RefugePreprocessing, args: argparse.Namespace
) -> RefugeSegmentation:
seg = RefugeSegmentation(pre)
seg.build_datasets(
image_size=args.seg_image_size,
batch_size=args.seg_batch_size,
num_workers=args.num_workers,
)
if not SEG_CKPT.exists():
raise FileNotFoundError(f"Segmentation checkpoint missing: {SEG_CKPT}")
state = torch.load(SEG_CKPT, map_location=args.device)
seg.model.load_state_dict(state)
seg.model.to(args.device)
return seg
def train_classifier(args: argparse.Namespace) -> None:
pre = ensure_preprocessing()
seg = _load_segmentation(pre, args)
backbone = build_classifier_backbone(args.clf_backbone)
print(f"[classifier] Using backbone: {args.clf_backbone}")
clf = RefugeClassification(
pre,
seg,
backbone=backbone,
cache_dir=args.clf_cache_dir,
use_all_labeled=args.clf_use_all,
auto_val_ratio=args.clf_auto_val_ratio,
)
clf.build_datasets(
crop_scale=args.crop_scale,
crop_size=args.crop_size,
batch_size=args.clf_batch_size,
num_workers=args.num_workers,
)
default_ckpt_path = classifier_checkpoint_path(args.clf_backbone)
ckpt_path = args.clf_checkpoint_path or default_ckpt_path
ckpt_dir = ckpt_path.parent
history = clf.train(
epochs=args.clf_epochs,
lr=args.clf_lr,
weight_decay=args.clf_weight_decay,
rotation_weight=args.rotation_weight,
checkpoint_dir=ckpt_dir,
device=args.device,
)
print("Classifier training complete. Best AUC:", history.get("best_auc"))
print(f"Checkpoint directory: {ckpt_dir}")
saved_path = ckpt_dir / "refuge_classifier_best.pt"
if ckpt_path != saved_path:
ckpt_path.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(saved_path, ckpt_path)
print(f"Checkpoint copied to: {ckpt_path}")
def _load_classifier(
pre: RefugePreprocessing, seg: RefugeSegmentation, args: argparse.Namespace
) -> Tuple[RefugeClassification, Path]:
backbone = build_classifier_backbone(args.clf_backbone)
clf = RefugeClassification(
pre,
seg,
backbone=backbone,
cache_dir=args.clf_cache_dir,
use_all_labeled=args.clf_use_all,
auto_val_ratio=args.clf_auto_val_ratio,
)
clf.build_datasets(
crop_scale=args.crop_scale,
crop_size=args.crop_size,
batch_size=args.clf_batch_size,
num_workers=args.num_workers,
)
ckpt_path = args.clf_checkpoint_path or classifier_checkpoint_path(
args.clf_backbone
)
if not ckpt_path.exists():
raise FileNotFoundError(f"Classifier checkpoint missing: {ckpt_path}")
print(f"[classifier] Loading checkpoint: {ckpt_path}")
state = torch.load(ckpt_path, map_location=args.device)
clf.backbone.load_state_dict(state["backbone"])
clf.classifier_head.load_state_dict(state["classifier"])
clf.rotation_head.load_state_dict(state["rotation"])
clf.backbone.to(args.device)
clf.classifier_head.to(args.device)
clf.rotation_head.to(args.device)
return clf, ckpt_path
def _collect_records(
pre: RefugePreprocessing,
seg: RefugeSegmentation,
clf: RefugeClassification,
dataset_name: str,
split: str,
scale: float,
) -> List[RefugeClassificationRecord]:
manifest = pre.build_manifest()
samples = [
sample
for sample in manifest
if sample.dataset == dataset_name
and sample.split == split
and sample.label is not None
]
if not samples:
return []
print(f"[eval] Preparing {len(samples)} samples for {dataset_name.upper()} {split}")
return clf.build_records_for_samples(
samples, crop_scale=scale, progress_prefix=f"{dataset_name}_{split}"
)
def _auc_for_records(
clf: RefugeClassification,
records: List[RefugeClassificationRecord],
device: str,
) -> float:
if not records:
return float("nan")
dataset = RefugeClassificationDataset(
records,
transform=clf.eval_transform,
polar_transform=clf.polar_transform,
size=clf.crop_size,
)
loader = DataLoader(dataset, batch_size=64, shuffle=False, num_workers=0)
clf.backbone.to(device).eval()
clf.classifier_head.to(device).eval()
preds: List[float] = []
targets: List[int] = []
with torch.no_grad():
for batch in tqdm(loader, desc="Eval", leave=False, unit="batch"):
images = batch["image"].to(device)
polars = batch["polar"].to(device)
extra_feats = batch["features"].to(device)
labels = batch["label"].cpu().numpy().tolist()
feats_img = clf.backbone(images)
feats = feats_img
if getattr(clf, "use_polar", False):
feats_polar = clf.backbone(polars)
feats = torch.cat([feats, feats_polar], dim=1)
if getattr(clf, "extra_feature_dim", 0) > 0:
feats = torch.cat([feats, extra_feats], dim=1)
logits = clf.classifier_head(feats)
probs = torch.softmax(logits, dim=1)[:, 1].cpu().numpy().tolist()
preds.extend(probs)
targets.extend(labels)
if len(set(targets)) < 2:
return float("nan")
return float(roc_auc_score(targets, preds))
def evaluate(args: argparse.Namespace) -> None:
pre = ensure_preprocessing()
seg = _load_segmentation(pre, args)
clf, clf_ckpt = _load_classifier(pre, seg, args)
def evaluate_subset(
clf_obj: RefugeClassification,
records: List[RefugeClassificationRecord],
label: str,
) -> None:
if not records:
print(f"[eval] No samples found for {label}; skipping.")
return
base_state = {
"backbone": clf_obj.backbone.state_dict(),
"rotation": clf_obj.rotation_head.state_dict(),
}
auc_no_ttt = _auc_for_records(clf_obj, records, device=args.device)
auc_ttt = float("nan")
if args.with_ttt:
ttt_loader = DataLoader(
RefugeClassificationDataset(
records,
transform=clf_obj.ttt_transform,
polar_transform=clf_obj.polar_transform,
size=clf_obj.crop_size,
),
batch_size=16,
shuffle=False,
num_workers=0,
)
ttt_iter = tqdm(range(args.ttt_steps), desc="TTT", unit="step")
for _ in ttt_iter:
clf_obj.apply_ttt(ttt_loader, device=args.device, steps=1)
auc_ttt = _auc_for_records(clf_obj, records, device=args.device)
clf_obj.backbone.load_state_dict(base_state["backbone"])
clf_obj.rotation_head.load_state_dict(base_state["rotation"])
print(
f"{label}: AUC (no TTT) = {auc_no_ttt:.4f}"
+ (f", AUC (TTT) = {auc_ttt:.4f}" if args.with_ttt else "")
)
if args.eval_datasets:
for dataset_name in dict.fromkeys(args.eval_datasets):
if dataset_name.lower() == "papila":
papila_records, papila_clf = build_papila_records(args, pre, clf_ckpt)
if papila_clf is None:
print("[eval] Papila evaluation aborted; no samples built.")
else:
evaluate_subset(papila_clf, papila_records, "PAPILA holdout")
else:
records = _collect_records(
pre,
seg,
clf,
dataset_name,
"holdout",
scale=args.crop_scale,
)
evaluate_subset(clf, records, f"{dataset_name.upper()} holdout")
return
# Do not mix splits: report per dataset + split
subsets = [
("refuge1", "val"),
("refuge2", "val"),
("refuge2", "test"),
]
for dataset_name, split in subsets:
records = _collect_records(
pre, seg, clf, dataset_name, split, scale=args.crop_scale
)
evaluate_subset(clf, records, f"{dataset_name.upper()} {split}")
if args.dump_masks and dataset_name == "refuge1" and split == "val":
out_dir = Path(args.dump_masks)
out_dir.mkdir(parents=True, exist_ok=True)
for rec in records:
sample = rec.sample
if sample is None:
continue
pred = seg.predict_mask(sample, device=args.device).numpy()
Image.fromarray((pred * 255).astype(np.uint8)).save(
out_dir / f"{sample.sample_id}_pred.png"
)
if sample.mask_path and sample.mask_path.exists():
Image.open(sample.mask_path).convert("L").save(
out_dir / f"{sample.sample_id}_gt.png"
)
def evaluate_segmentation(args: argparse.Namespace) -> None:
manifest_path = args.seg_manifest or Path("manifest.csv")
mask_cache_dir = None if args.in_memory_cache else args.mask_cache_dir
image_cache_dir = None if args.in_memory_cache else args.image_cache_dir
segmenter = UNetSegmenter(
manifest_path=manifest_path,
normalize=args.seg_normalize,
device=args.device,
mask_cache_dir=mask_cache_dir,
image_cache_dir=image_cache_dir,
in_memory_cache=args.in_memory_cache,
loader_workers=args.loader_workers,
)
if args.in_memory_cache:
segmenter.prebuild_in_memory_cache(
cache_workers=max(0, int(args.cache_workers)),
include_train=False,
include_val=bool(args.eval_seg_splits is None or "val" in args.eval_seg_splits),
include_holdout=bool(args.eval_seg_splits is None or "holdout" in args.eval_seg_splits),
)
ckpt = resolve_unet_weights(args.seg_weights)
if ckpt.exists():
state = torch.load(ckpt, map_location=segmenter.device)
state_dict = state.get("model", state)
segmenter.model.load_state_dict(state_dict, strict=False)
print(f"[seg-eval] Loaded weights from {ckpt}")
else:
raise FileNotFoundError(f"Segmentation weights not found at {ckpt}")
dataset_filter = args.eval_seg_datasets
split_filter = args.eval_seg_splits
output_dir = args.eval_seg_output or Path("analysis_data/segmenter_eval")
metrics_path = args.eval_seg_metrics_path
segmenter.evaluate_dataset(
dataset_filter=dataset_filter,
split_filter=split_filter,
output_dir=output_dir,
save_overlays=not args.eval_seg_no_overlays,
metrics_path=metrics_path,
threshold=args.eval_seg_threshold,
tta=args.eval_seg_tta,
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="REFUGE pipeline helper")
parser.add_argument(
"--train-seg", action="store_true", help="Train the segmentation model"
)
parser.add_argument(
"--train-unet-seg",
action="store_true",
help="Train the UNet segmenter (replacement for scripts/run_unet_segmenter.py)",
)
parser.add_argument(
"--train-clf", action="store_true", help="Train the classification model"
)
parser.add_argument(
"--eval", action="store_true", help="Run evaluation on stored checkpoints"
)
parser.add_argument(
"--with-ttt",
action="store_true",
help="Apply test-time training during evaluation",
)
parser.add_argument(
"--ttt-steps", type=int, default=1, help="TTT epochs over evaluation loader"
)
parser.add_argument(
"--export-backbone",
type=Path,
default=None,
help="Optional path to export the trained backbone weights",
)
parser.add_argument(
"--dump-masks",
type=Path,
default=None,
help="Optional directory to dump predicted/GT masks during eval",
)
parser.add_argument(
"--device", default="cuda" if torch.cuda.is_available() else "cpu"
)
parser.add_argument("--num-workers", type=int, default=4)
# Segmentation hyperparameters
parser.add_argument("--seg-epochs", type=int, default=40)
parser.add_argument("--seg-lr", type=float, default=1e-3)
parser.add_argument("--seg-weight-decay", type=float, default=1e-5)
parser.add_argument("--seg-image-size", type=int, default=512)
parser.add_argument("--seg-batch-size", type=int, default=4)
parser.add_argument(
"--seg-manifest",
type=Path,
default=Path("manifest.csv"),
help="Manifest CSV for the UNet segmenter (default: manifest.csv)",
)
parser.add_argument(
"--seg-weights",
type=Path,
default=None,
help="Path to UNet segmenter weights (default: models/unet_segmenter/best.pt)",
)
parser.add_argument(
"--seg-normalize",
choices=["none", "imagenet", "per_image"],
default="none",
help="Normalization mode used when running the UNet segmenter",
)
parser.add_argument(
"--seg-strong-aug",
action="store_true",
help="Enable stronger geometric augmentations when training the UNet segmenter",
)
parser.add_argument(
"--seg-train-datasets",
nargs="+",
default=["refuge"],
help="Datasets to use for UNet segmenter training (default: refuge)",
)
parser.add_argument(
"--seg-val-datasets",
nargs="+",
default=["refuge"],
help="Datasets eligible for validation sampling (default: refuge)",
)
parser.add_argument(
"--seg-holdout-datasets",
nargs="+",
default=["refuge"],
help="Datasets reserved for holdout set during UNet segmenter training (default: refuge)",
)
parser.add_argument(
"--seg-checkpoint-dir",
type=Path,
default=Path("models/v2/refuge/segmentation/per_image"),
help="Directory to store UNet segmenter checkpoints",
)
parser.add_argument(
"--loader-workers",
type=int,
default=0,
help="DataLoader workers for UNet segmenter train/eval.",
)
parser.add_argument(
"--mask-cache-dir",
type=Path,
default=None,
help="Optional cache dir for parsed/resized disc+cup masks.",
)
parser.add_argument(
"--image-cache-dir",
type=Path,
default=None,
help="Optional cache dir for resized RGB images before augmentation.",
)
parser.add_argument(
"--in-memory-cache",
action="store_true",
help="Cache preprocessed images and masks in RAM (per DataLoader worker process).",
)
parser.add_argument(
"--cache-workers",
type=int,
default=0,
help="Worker threads for prebuilding in-memory cache before training/eval.",
)
# Classification hyperparameters
parser.add_argument("--clf-epochs", type=int, default=30)
parser.add_argument("--clf-lr", type=float, default=1e-4)
parser.add_argument("--clf-weight-decay", type=float, default=1e-4)
parser.add_argument("--clf-batch-size", type=int, default=16)
parser.add_argument(
"--clf-backbone",
choices=sorted(CLASSIFIER_BACKBONES.keys()),
default="resnet50",
help="Backbone architecture for the REFUGE classifier",
)
parser.add_argument("--rotation-weight", type=float, default=0.5)
parser.add_argument("--crop-scale", type=float, default=2.5)
parser.add_argument("--crop-size", type=int, default=224)
parser.add_argument(
"--clf-cache-dir",
type=Path,
default=Path("analysis_data/classifier_cache"),
help="Directory to cache classifier preprocessing artifacts",
)
parser.add_argument(
"--clf-use-all",
action="store_true",
help="Use all labelled samples (train+val) when building classifier dataset",
)
parser.add_argument(
"--clf-auto-val-ratio",
type=float,
default=0.1,
help="Fraction for automatic validation split when no explicit val set is used",
)
parser.add_argument(
"--clf-checkpoint-path",
type=Path,
default=None,
help="Optional explicit path for the classifier checkpoint (defaults to models/refuge/classifier/<backbone>/refuge_classifier_best.pt)",
)
parser.add_argument(
"--eval-datasets",
nargs="+",
help="Datasets to evaluate during --eval (e.g. papila). Defaults to REFUGE splits.",
)
# Segmentation evaluation parameters
parser.add_argument(
"--eval-seg",
action="store_true",
help="Evaluate the segmentation model on specified datasets/splits",
)
parser.add_argument(
"--eval-seg-datasets",
nargs="+",
default=["refuge"],
help="Segmentation datasets to evaluate (default: refuge)",
)
parser.add_argument(
"--eval-seg-splits",
nargs="+",
choices=["train", "val", "holdout"],
help="Segmentation splits to evaluate (default: val)",
)
parser.add_argument(
"--eval-seg-output",
type=Path,
default=Path("analysis_data/segmenter_eval"),
help="Directory to store segmentation metrics CSVs",
)
parser.add_argument(
"--eval-seg-threshold",
type=float,
default=0.5,
help="Threshold for binarising predicted masks during segmentation eval",
)
parser.add_argument(
"--eval-seg-metrics-path",
type=Path,
default=None,
help="Optional explicit CSV path for segmentation metrics output",
)
parser.add_argument(
"--eval-seg-no-overlays",
action="store_true",
help="Skip saving GT/pred overlay images during segmentation evaluation",
)
parser.add_argument(
"--eval-seg-tta",
action="store_true",
help="Enable horizontal/vertical flip TTA during segmentation evaluation",
)
parser.add_argument(
"--papila-metrics",
type=Path,
default=None,
help="Optional CSV of Papila Dice metrics used to filter samples",
)
parser.add_argument(
"--papila-dice-threshold",
type=float,
default=0.01,
help="Minimum Dice required (disc or cup) when filtering Papila metrics",
)
parser.add_argument(
"--papila-positive-labels",
nargs="+",
default=["glaucoma", "glaucoma suspect", "suspect"],
help="Papila label values treated as positive when labels are non-numeric",
)
parser.add_argument(
"--papila-image-dir",
type=Path,
default=Path("Papila/FundusImages"),
help="Path to Papila fundus images",
)
parser.add_argument(
"--papila-clinical-dir",
type=Path,
default=Path("Papila/ClinicalData"),
help="Path to Papila clinical CSVs",
)
parser.add_argument(
"--papila-label-col",
type=str,
default="Diagnosis",
help="Column name containing Papila labels",
)
parser.add_argument(
"--papila-use-gt",
action="store_true",
help="Use Papila ground-truth contours when evaluating classifiers",
)
parser.add_argument(
"--papila-contours-dir",
type=Path,
default=Path("Papila/ExpertsSegmentations/Contours"),
help="Directory containing Papila contour text files",
)
return parser.parse_args()
def main() -> None:
args = parse_args()
if not any(
[
args.train_seg,
args.train_unet_seg,
args.train_clf,
args.eval,
args.eval_seg,
args.export_backbone,
]
):
raise SystemExit(
"Specify at least one action: --train-seg, --train-unet-seg, --train-clf, --eval, --eval-seg, or --export-backbone"
)
if args.train_seg:
train_segmentation(args)
if args.train_unet_seg:
train_unet_segmenter(args)
if args.train_clf:
train_classifier(args)
if args.eval:
evaluate(args)
if args.eval_seg:
evaluate_segmentation(args)
if args.export_backbone:
pre = ensure_preprocessing()
seg = _load_segmentation(pre, args)
clf = _load_classifier(pre, seg, args)
out_path = args.export_backbone
out_path.parent.mkdir(parents=True, exist_ok=True)
torch.save(clf.extract_backbone().state_dict(), out_path)
print(f"Backbone weights exported to {out_path}")
if __name__ == "__main__":
main()
scripts/main/refuge/run_unet_segmenter.py
Executable
+156
View File
@@ -0,0 +1,156 @@
#!/usr/bin/env python3
"""Train and evaluate the U-Net optic disc/cup segmenter."""
from __future__ import annotations
import argparse
from pathlib import Path
import torch
import sys
REPO_ROOT = Path(__file__).resolve().parents[3]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.unet_segmenter import UNetSegmenter
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="UNet segmenter runner")
parser.add_argument("--manifest", type=Path, required=True, help="Path to manifest CSV")
parser.add_argument("--train", action="store_true", help="Train the segmenter")
parser.add_argument("--evaluate", action="store_true", help="Evaluate on holdout set")
parser.add_argument("--epochs", type=int, default=40)
parser.add_argument("--batch-size", type=int, default=4)
parser.add_argument("--lr", type=float, default=1e-3)
parser.add_argument("--weight-decay", type=float, default=1e-5)
parser.add_argument("--disc-weight", type=float, default=1.0)
parser.add_argument("--cup-weight", type=float, default=1.0)
parser.add_argument("--checkpoint-dir", type=Path, default=Path("models/unet_segmenter"))
parser.add_argument("--eval-output", type=Path, default=Path("analysis_data/segmenter_eval"))
parser.add_argument(
"--normalize",
choices=["none", "imagenet", "per_image"],
default="none",
help="Image normalization mode for train/eval",
)
parser.add_argument(
"--strong-aug",
action="store_true",
help="Enable stronger train-time augmentations (flips/rotations)",
)
parser.add_argument("--train-datasets", nargs="+", help="Datasets to use for training/validation (default: all)")
parser.add_argument("--val-datasets", nargs="+", help="Datasets eligible for validation sampling (default: match training)")
parser.add_argument("--holdout-datasets", nargs="+", help="Restrict holdout entries to these datasets (default: all)")
parser.add_argument(
"--val-ratio",
type=float,
default=0.1,
help="Fraction of training data reserved for validation (default: 0.1)",
)
parser.add_argument("--eval-datasets", nargs="+", help="Datasets to evaluate (default: holdout split only)")
parser.add_argument("--eval-splits", nargs="+", help="Splits to evaluate (default: holdout or all when --eval-datasets is set)")
parser.add_argument("--eval-metrics-path", type=Path, help="Optional CSV path for evaluation metrics output")
parser.add_argument("--no-eval-overlays", action="store_true", help="Skip writing overlay images during evaluation")
parser.add_argument("--threshold", type=float, default=0.5, help="Probability threshold for binarizing predictions")
parser.add_argument("--tta", action="store_true", help="Enable simple test-time augmentation (H/V flips) during evaluation")
parser.add_argument(
"--weights",
type=Path,
help="Optional model weights (.pt) for eval-only runs; defaults to <checkpoint-dir>/best.pt",
)
parser.add_argument("--device", choices=["auto", "cuda", "cpu"], default="auto", help="Execution device for UNet (default: auto).")
parser.add_argument("--loader-workers", type=int, default=0, help="DataLoader workers for train/eval.")
parser.add_argument("--mask-cache-dir", type=Path, default=None, help="Optional cache dir for parsed/resized disc+cup masks.")
parser.add_argument("--image-cache-dir", type=Path, default=None, help="Optional cache dir for resized RGB images before augmentation.")
parser.add_argument("--in-memory-cache", action="store_true", help="Cache preprocessed images and masks in RAM (per DataLoader worker process).")
parser.add_argument("--cache-workers", type=int, default=0, help="Worker threads for prebuilding in-memory cache before training/eval.")
return parser.parse_args()
def main() -> None:
args = parse_args()
if args.device == "auto":
selected_device = "cuda" if torch.cuda.is_available() else "cpu"
else:
selected_device = args.device
if selected_device == "cuda" and not torch.cuda.is_available():
raise RuntimeError("Requested --device cuda but CUDA is not available.")
print(
f"[UNet] device={selected_device} "
f"(cuda_available={torch.cuda.is_available()}, workers={args.loader_workers})"
)
if selected_device == "cuda":
idx = torch.cuda.current_device()
print(f"[UNet] gpu={torch.cuda.get_device_name(idx)}")
mask_cache_dir = None if args.in_memory_cache else args.mask_cache_dir
image_cache_dir = None if args.in_memory_cache else args.image_cache_dir
if args.in_memory_cache and (args.mask_cache_dir or args.image_cache_dir):
print("[UNet] in_memory_cache enabled: disk caches disabled for this run.")
segmenter = UNetSegmenter(
manifest_path=args.manifest,
device=selected_device,
cup_weight=args.cup_weight,
disc_weight=args.disc_weight,
val_ratio=args.val_ratio,
train_datasets=args.train_datasets,
val_datasets=args.val_datasets,
holdout_datasets=args.holdout_datasets,
normalize=args.normalize,
use_stronger_aug=args.strong_aug,
mask_cache_dir=mask_cache_dir,
image_cache_dir=image_cache_dir,
in_memory_cache=args.in_memory_cache,
loader_workers=args.loader_workers,
)
if mask_cache_dir:
print(f"[UNet] mask_cache_dir={mask_cache_dir}")
if image_cache_dir:
print(f"[UNet] image_cache_dir={image_cache_dir}")
if args.in_memory_cache:
print("[UNet] in_memory_cache=enabled (note: memory use scales with loader workers)")
segmenter.prebuild_in_memory_cache(
cache_workers=max(0, int(args.cache_workers)),
include_train=bool(args.train),
include_val=bool(args.train),
include_holdout=bool(args.evaluate),
)
if args.train:
segmenter.train(
epochs=args.epochs,
batch_size=args.batch_size,
lr=args.lr,
weight_decay=args.weight_decay,
checkpoint_dir=args.checkpoint_dir,
)
if args.evaluate:
if not args.train:
ckpt = args.weights or (args.checkpoint_dir / "best.pt")
if ckpt and ckpt.exists():
state = torch.load(ckpt, map_location=segmenter.device)
state_dict = state.get("model", state)
segmenter.model.load_state_dict(state_dict, strict=False)
print(f"Loaded weights from {ckpt}")
else:
print(f"[warn] No checkpoint found at {ckpt}. Evaluating untrained weights.")
split_filter = {"holdout"} if args.eval_splits is None else args.eval_splits
segmenter.evaluate_dataset(
dataset_filter=args.eval_datasets,
split_filter=split_filter,
output_dir=args.eval_output,
save_overlays=not args.no_eval_overlays,
metrics_path=args.eval_metrics_path,
threshold=args.threshold,
tta=args.tta,
)
if __name__ == "__main__":
main()
scripts/main/v1/run_multifold.py
Executable
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env python3
"""CLI wrapper that delegates to classes.frontend.Multifold."""
from pathlib import Path
import sys
# ensure repo root on path
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.frontend import Multifold
def run_cli(cli_args=None):
parser = Multifold.build_parser()
args = parser.parse_args(cli_args)
runner = Multifold(args)
runner.run()
def main():
run_cli()
if __name__ == "__main__":
main()
scripts/main/v1/run_multifold_grid.py
Executable
+431
View File
@@ -0,0 +1,431 @@
#!/usr/bin/env python3
"""
Grid-search runner for run_multifold experiments.
Features:
* Enumerates the requested configuration grid and writes grid_plan.csv.
* Picks the next incomplete run, marks it running, executes run_multifold.py.
* Records AUC/accuracy metrics per fold into grid_report.csv.
* Removes model checkpoints for runs dominated (80%+ metrics worse) by others.
"""
from __future__ import annotations
import argparse
import csv
import json
import math
import os
import shutil
import subprocess
import sys
from contextlib import contextmanager
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional
import fcntl
REPO_ROOT = Path(__file__).resolve().parents[2]
RUN_SCRIPT = REPO_ROOT / "scripts" / "run_multifold.py"
MANIFEST = REPO_ROOT / "manifest.csv"
GRID_DIR = REPO_ROOT / "analysis_data" / "grid_search"
PLAN_PATH = GRID_DIR / "grid_plan.csv"
REPORT_PATH = GRID_DIR / "grid_report.csv"
LOCK_PATH = GRID_DIR / ".grid_lock"
MODELS_ROOT = REPO_ROOT / "models" / "grid_search"
def parse_args() -> argparse.Namespace:
ap = argparse.ArgumentParser(description="Grid-search orchestrator for run_multifold.")
ap.add_argument("--plan-date", default=datetime.now().strftime("%Y%m%d"),
help="Date prefix used when generating run IDs (default: today).")
ap.add_argument("--regen-plan", action="store_true",
help="Rebuild the grid plan from scratch (overwrites existing plan).")
ap.add_argument("--manifest", type=Path, default=MANIFEST,
help="UNet manifest CSV for cropper.")
ap.add_argument("--weights-dir", type=Path, default=REPO_ROOT / "models" / "unet_segmenter",
help="Directory containing norm_* subfolders with best.pt.")
ap.add_argument("--dry-run", action="store_true", help="Enumerate next run without executing.")
ap.add_argument("--max-runs", type=int, default=1,
help="Maximum runs to execute in this invocation (default: 1).")
ap.add_argument("--run-all", action="store_true",
help="Execute runs sequentially until plan is exhausted (overrides --max-runs).")
return ap.parse_args()
@contextmanager
def file_lock(lock_path: Path):
lock_path.parent.mkdir(parents=True, exist_ok=True)
with open(lock_path, "w") as lock_file:
fcntl.flock(lock_file, fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(lock_file, fcntl.LOCK_UN)
def read_csv(path: Path) -> List[Dict[str, str]]:
if not path.exists():
return []
with path.open(newline="") as fh:
reader = csv.DictReader(fh)
return list(reader)
def write_csv(path: Path, rows: List[Dict[str, str]], headers: List[str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", newline="") as fh:
writer = csv.DictWriter(fh, fieldnames=headers)
writer.writeheader()
for row in rows:
writer.writerow(row)
def grid_configs(base_date: str, weights_dir: Path) -> List[Dict[str, str]]:
eval_modes = ["binary", "multiclass"]
crop_variants = [
("norm_imagenet", "imagenet"),
("normalize_none", "none"),
("norm_per_image", "per_image"),
]
tta_opts = [False, True]
loss_modes = ["focal", "balanced", "none"]
thaw_modes = ["none", "gradual"]
se_configs = []
# none
se_configs.append(("none", {"se_enabled": False}))
# bridge only
for pre in (True, False):
se_configs.append((
"bridge",
{"se_enabled": True, "se_where": "bridge", "bridge_pre_norm": pre, "tower_pre_norm": None},
))
# tower only
for pre in (True, False):
se_configs.append((
"tower",
{"se_enabled": True, "se_where": "tower", "bridge_pre_norm": None, "tower_pre_norm": pre},
))
# both (four combos)
for b_pre in (True, False):
for t_pre in (True, False):
se_configs.append((
"both",
{
"se_enabled": True,
"se_where": "both",
"bridge_pre_norm": b_pre,
"tower_pre_norm": t_pre,
},
))
combos = []
idx = 0
for eval_mode in eval_modes:
for variant, norm in crop_variants:
weights_path = weights_dir / variant / "best.pt"
for tta in tta_opts:
for loss in loss_modes:
for thaw in thaw_modes:
for se_name, se_opts in se_configs:
run_id = f"{base_date}-{idx:04d}"
combos.append({
"run_id": run_id,
"status": "incomplete",
"eval_mode": eval_mode,
"crop_variant": variant,
"crop_normalize": norm,
"crop_weights": str(weights_path),
"crop_tta": str(tta),
"loss_mode": loss,
"thaw_mode": thaw,
"se_mode": se_name,
"se_bridge_pre_norm": str(se_opts.get("bridge_pre_norm")),
"se_tower_pre_norm": str(se_opts.get("tower_pre_norm")),
})
idx += 1
return combos
PLAN_HEADERS = [
"run_id",
"status",
"eval_mode",
"crop_variant",
"crop_normalize",
"crop_weights",
"crop_tta",
"loss_mode",
"thaw_mode",
"se_mode",
"se_bridge_pre_norm",
"se_tower_pre_norm",
]
def ensure_plan(args: argparse.Namespace) -> None:
if args.regen_plan or not PLAN_PATH.exists():
combos = grid_configs(args.plan_date, args.weights_dir)
write_csv(PLAN_PATH, combos, PLAN_HEADERS)
print(f"[grid] Plan created with {len(combos)} runs at {PLAN_PATH}")
def select_next_run() -> Optional[Dict[str, str]]:
rows = read_csv(PLAN_PATH)
for row in rows:
if row["status"] == "incomplete":
row["status"] = "running"
write_csv(PLAN_PATH, rows, PLAN_HEADERS)
return row
return None
def update_run_status(run_id: str, new_status: str) -> None:
rows = read_csv(PLAN_PATH)
for row in rows:
if row["run_id"] == run_id:
row["status"] = new_status
break
write_csv(PLAN_PATH, rows, PLAN_HEADERS)
def build_run_command(row: Dict[str, str], manifest: Path) -> List[str]:
cmd = [
sys.executable,
str(RUN_SCRIPT),
"--backbone",
"resnet50",
"--fusion-mode",
"fused",
"--epochs",
"40",
"--batch-size",
"8",
"--img-crop-manifest",
str(manifest),
"--img-crop-weights",
row["crop_weights"],
"--img-crop-normalize",
row["crop_normalize"],
"--eval_mode",
row["eval_mode"],
"--holdout-per-class",
"12",
"--run-id",
row["run_id"],
"--shortname",
"grid_search",
]
if row["crop_tta"] == "True":
cmd.append("--img-crop-tta")
# Loss/balancing modes
if row["loss_mode"] == "focal":
cmd.extend(["--focal-gamma", "2.0"])
elif row["loss_mode"] == "balanced":
cmd.append("--balanced-sampler")
# Thaw schedule
if row["thaw_mode"] == "gradual":
cmd.append("--gradual-thaw")
cmd.extend(["--thaw-ratio", "0.33"])
cmd.extend(["--thaw-start-epoch", "10"])
cmd.extend(["--thaw-target", "image"])
# SE settings
if row["se_mode"] == "none":
cmd.append("--no-se")
else:
cmd.extend(["--se-reduction", "16"])
cmd.extend(["--se-reduction-tower", "16"])
cmd.extend(["--se-where", row["se_mode"]])
bridge_pre = row["se_bridge_pre_norm"]
tower_pre = row["se_tower_pre_norm"]
if bridge_pre == "True":
cmd.append("--se-pre-norm")
elif bridge_pre == "False":
cmd.append("--no-se-pre-norm")
if tower_pre == "True":
cmd.append("--se-pre-norm-tower")
elif tower_pre == "False":
cmd.append("--no-se-pre-norm-tower")
return cmd
def run_command(cmd: List[str]) -> None:
print("[grid] Launching:", " ".join(cmd))
subprocess.run(cmd, check=True)
METRIC_KEYS = ["auc_fused", "auc_img", "auc_md", "acc_fused", "acc_img", "acc_md"]
def extract_metrics(run_id: str) -> Dict[str, str]:
summary_path = REPO_ROOT / "analysis_data" / "grid_search" / run_id / "summary.json"
if not summary_path.exists():
raise FileNotFoundError(f"Missing summary.json for run {run_id}")
with summary_path.open() as fh:
summary = json.load(fh)
rows = {}
for fold in summary.get("fold_metrics", []):
if not isinstance(fold, dict):
continue
f_idx = fold.get("fold")
stats = fold.get("stats") or {}
if not isinstance(stats, dict):
continue
for key in METRIC_KEYS:
val = stats.get(key)
if val is None:
continue
rows[f"metric_fold{f_idx}_{key}"] = str(val)
best_mean = summary.get("best_metric_mean")
if best_mean is not None:
rows["metric_best_mean"] = str(best_mean)
return rows
def update_report(row: Dict[str, str], metrics: Dict[str, str]) -> None:
existing = read_csv(REPORT_PATH)
# Remove existing entry for run_id
existing = [r for r in existing if r.get("run_id") != row["run_id"]]
record = {**row, **metrics}
existing.append(record)
headers = sorted({key for r in existing for key in r.keys()})
write_csv(REPORT_PATH, existing, headers)
def load_report_rows() -> List[Dict[str, str]]:
return read_csv(REPORT_PATH)
def metric_columns(rows: List[Dict[str, str]]) -> List[str]:
keys = set()
for row in rows:
for key in row:
if key.startswith("metric_"):
keys.add(key)
return sorted(keys)
def _to_float(val: str) -> Optional[float]:
try:
f = float(val)
if math.isnan(f):
return None
return f
except Exception:
return None
def prune_dominated(rows: List[Dict[str, str]]) -> None:
"""
Remove model directories for runs that are clearly dominated by another run.
A run is dominated if:
* Another run has a strictly higher metric_best_mean, OR
* Another run is >= on >=80% of overlapping metrics and strictly better on at least one.
"""
metrics = metric_columns(rows)
if not metrics:
return
dominated = set()
for row in rows:
run_id = row["run_id"]
row_vals = {m: row.get(m) for m in metrics}
row_best = _to_float(row_vals.get("metric_best_mean"))
for other in rows:
if other["run_id"] == run_id:
continue
other_vals = {m: other.get(m) for m in metrics}
other_best = _to_float(other_vals.get("metric_best_mean"))
# Fast path: compare aggregate best mean if both have it
if row_best is not None and other_best is not None and other_best > row_best:
dominated.add(run_id)
break
# Fallback: overlap-wise dominance
comparisons = []
better = 0
for key in metrics:
v1 = _to_float(row_vals.get(key))
v2 = _to_float(other_vals.get(key))
if v1 is None or v2 is None:
continue
comparisons.append(v2 >= v1)
if v2 > v1:
better += 1
if not comparisons:
continue
fraction = sum(comparisons) / len(comparisons)
if fraction >= 0.8 and better > 0:
dominated.add(run_id)
break
for run_id in dominated:
model_dir = MODELS_ROOT / run_id
if model_dir.exists():
print(f"[grid] Removing dominated model artifacts for {run_id}")
try:
shutil.rmtree(model_dir)
except OSError as exc:
# Don't fail the grid run if cleanup isn't permitted (e.g., locked SMB dirs).
print(f"[grid] Warning: could not remove {model_dir}: {exc}")
def main():
args = parse_args()
ensure_plan(args)
if args.dry_run:
with file_lock(LOCK_PATH):
next_run = select_next_run()
if next_run is None:
print("[grid] No incomplete runs remaining.")
return
update_run_status(next_run["run_id"], "incomplete")
print("[grid] Next run:", next_run)
return
max_runs = None if args.run_all else args.max_runs
runs_done = 0
while True:
with file_lock(LOCK_PATH):
next_run = select_next_run()
if next_run is None:
if runs_done == 0:
print("[grid] All runs completed.")
else:
print(f"[grid] No more runs remaining after {runs_done} run(s).")
return
run_id = next_run["run_id"]
try:
cmd = build_run_command(next_run, args.manifest)
run_command(cmd)
metrics = extract_metrics(run_id)
with file_lock(LOCK_PATH):
update_run_status(run_id, "completed")
update_report(next_run, metrics)
report_rows = load_report_rows()
prune_dominated(report_rows)
print(f"[grid] Run {run_id} completed.")
except Exception as exc:
with file_lock(LOCK_PATH):
update_run_status(run_id, "incomplete")
raise SystemExit(f"[grid] Run {run_id} failed: {exc}") from exc
runs_done += 1
if max_runs is not None and runs_done >= max_runs:
print(f"[grid] Reached run limit ({max_runs}); stopping.")
return
if __name__ == "__main__":
main()
scripts/main/v2/run_multifold_v2.py
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python3
"""CLI wrapper that delegates to classes.frontend.Multifold with V2 loaders."""
from __future__ import annotations
from pathlib import Path
import sys
# ensure repo root on path
REPO_ROOT = Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
import classes.frontend as frontend
from classes.v2.v2_hypertower import V2HyperTower
def run_cli(cli_args=None):
parser = frontend.Multifold.build_parser()
parser.set_defaults(warmup_tower_epochs=None, warmup_fused_epochs=None)
parser.add_argument(
"--sample-mode",
choices=["eye", "patient"],
default="eye",
help="Build samples per eye (row-level) or per patient (multi-slot).",
)
args = parser.parse_args(cli_args)
# Monkeypatch the HyperTower class used inside Multifold.
frontend.HyperTower = V2HyperTower
runner = frontend.Multifold(args)
runner.run()
def main():
run_cli()
if __name__ == "__main__":
main()
scripts/main/v2/run_multifold_v2_modes.py
+21
View File
@@ -0,0 +1,21 @@
#!/usr/bin/env python3
"""CLI wrapper for the V2 three-mode comparison (classic/ensemble/bilateral)."""
from __future__ import annotations
from pathlib import Path
import sys
REPO_ROOT = Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes.v2.v2_hypertower import V2ModeComparator
def main():
V2ModeComparator.run()
if __name__ == "__main__":
main()
scripts/output_analysis/diagnostics/fold_confusion_matrix.py
Executable
+165
View File
@@ -0,0 +1,165 @@
#!/usr/bin/env python3
"""
Inspect saved validation/holdout logits for a multifold run.
Prints per-class AUCs and sample counts so we can sanity-check unusually high scores.
Can also print per-fold confusion matrices.
Example:
python scripts/fold_confusion_matrix.py \
--run-dir analysis_data/1030_Balanced_Unet_Perimg_Resnet_SE16NormB_SE16NormT_multi_fused/1030_Balanced_Unet_Perimg_Resnet_SE16NormB_SE16NormT_multi_fused_20251030_091842 \
--head fused
python scripts/fold_confusion_matrix.py --run-dir ... --head fused --use-holdout --confusion
"""
from __future__ import annotations
import argparse
import json
import math
from pathlib import Path
from typing import Dict, List
import numpy as np
from sklearn.metrics import roc_auc_score, confusion_matrix
def parse_args() -> argparse.Namespace:
ap = argparse.ArgumentParser(description="Inspect saved logits for a run and report per-class AUCs.")
ap.add_argument("--run-dir", required=True, type=Path, help="Path to the run directory under analysis_data.")
ap.add_argument("--head", choices=["fused", "image", "metadata"], default="fused",
help="Which prediction head's saved probabilities to load.")
ap.add_argument("--use-holdout", action="store_true",
help="Look for *_holdout.npy dumps instead of validation splits.")
ap.add_argument("--class-names", nargs="*", default=None,
help="Optional override for class labels (order should match numeric labels).")
ap.add_argument("--macro", action="store_true", help="Also print macro-average AUC across classes.")
ap.add_argument("--confusion", action="store_true", help="Print confusion matrix for each fold.")
return ap.parse_args()
def load_cli_args(run_dir: Path) -> Dict:
path = run_dir / "cli_args.json"
if not path.exists():
raise FileNotFoundError(f"Missing cli_args.json in {run_dir}")
with path.open("r", encoding="utf-8") as fh:
return json.load(fh)
def find_fold_files(run_dir: Path, suffix: str) -> Dict[int, Dict[str, Path]]:
files: Dict[int, Dict[str, Path]] = {}
for y_file in run_dir.glob(f"fold*_y_true{suffix}.npy"):
fold_str = y_file.stem.split("_")[0].replace("fold", "")
try:
fold_idx = int(fold_str)
except ValueError:
continue
files.setdefault(fold_idx, {})["y_true"] = y_file
for head_key, glob_pat in [
("fused", f"fold*_probs_fused{suffix}.npy"),
("image", f"fold*_probs_img{suffix}.npy"),
("metadata", f"fold*_probs_md{suffix}.npy"),
]:
for p_file in run_dir.glob(glob_pat):
fold_str = p_file.stem.split("_")[0].replace("fold", "")
try:
fold_idx = int(fold_str)
except ValueError:
continue
files.setdefault(fold_idx, {})[head_key] = p_file
return files
def compute_auc(y_true: np.ndarray, probs: np.ndarray, class_names: List[str], macro: bool) -> List[int]:
num_classes = probs.shape[1]
unique = np.unique(y_true)
print(f" classes present: {sorted(unique.tolist())}")
aucs = []
seen_classes: List[int] = []
for cls in range(num_classes):
name = class_names[cls] if cls < len(class_names) else f"class_{cls}"
mask = (y_true == cls)
pos = int(mask.sum())
neg = len(y_true) - pos
if pos == 0 or neg == 0:
print(f" {name:<15} -> insufficient positives/negatives (pos={pos}, neg={neg}); skipping AUC")
continue
try:
auc = roc_auc_score((y_true == cls).astype(int), probs[:, cls])
except ValueError as exc:
print(f" {name:<15} -> AUC error: {exc}")
continue
aucs.append(auc)
seen_classes.append(cls)
print(f" {name:<15} -> AUC={auc:.4f} (pos={pos}, neg={neg})")
if macro and aucs:
mean = float(np.mean(aucs))
std = float(np.std(aucs, ddof=0)) if len(aucs) > 1 else math.nan
print(f" macro AUC across reported classes: {mean:.4f} (std={std:.4f})")
return seen_classes
def print_confusion(y_true: np.ndarray, probs: np.ndarray, class_names: List[str]) -> None:
num_classes = probs.shape[1]
preds = probs.argmax(axis=1)
labels = list(range(num_classes))
cm = confusion_matrix(y_true, preds, labels=labels)
names = [class_names[i] if i < len(class_names) else f"class_{i}" for i in labels]
header = " " * 14 + "".join(f"{name:>12}" for name in names)
print(" Confusion matrix (rows=true, cols=pred):")
print(header)
for idx, row in enumerate(cm):
label = names[idx]
row_str = "".join(f"{int(val):>12}" for val in row)
print(f" {label:<12}{row_str}")
def main() -> None:
args = parse_args()
run_dir = args.run_dir.resolve()
if not run_dir.exists():
raise FileNotFoundError(run_dir)
cli_args = load_cli_args(run_dir)
eval_mode = cli_args.get("eval_mode", "multiclass")
if args.class_names:
class_names = args.class_names
else:
if eval_mode == "binary":
class_names = ["Healthy", "Glaucoma"]
else:
class_names = cli_args.get("class_names") or ["Healthy", "Glaucoma", "Suspect"]
suffix = "_holdout" if args.use_holdout else ""
files = find_fold_files(run_dir, suffix)
if not files:
raise SystemExit(f"No saved probability files matching suffix '{suffix}' found in {run_dir}. "
"Run scripts/rebuild_run_best_plots.py first if needed.")
print(f"[info] Inspecting head='{args.head}' ({'holdout' if args.use_holdout else 'validation'})")
for fold_idx in sorted(files.keys()):
fold = files[fold_idx]
if "y_true" not in fold:
print(f"[warning] Fold {fold_idx}: missing y_true file; skipping.")
continue
head_key = {
"fused": "fused",
"image": "image",
"metadata": "metadata",
}[args.head]
prob_path = fold.get(head_key)
if prob_path is None:
print(f"[warning] Fold {fold_idx}: missing probability file for head '{args.head}'; skipping.")
continue
y_true = np.load(fold["y_true"])
probs = np.load(prob_path)
print(f"\n Fold {fold_idx} -> samples={len(y_true)} file={prob_path.name}")
compute_auc(y_true, probs, class_names, args.macro)
if args.confusion:
print_confusion(y_true, probs, class_names)
if __name__ == "__main__":
main()
scripts/output_analysis/grid_search/batch_best_metrics.py
Executable
+356
View File
@@ -0,0 +1,356 @@
#!/usr/bin/env python3
"""
Scan an analysis directory for HyperTower run folders, extract the best per-fold
metric/accuracy from the epoch logs, and emit a combined summary.
Example:
python scripts/batch_best_metrics.py \
--analysis-dir analysis_data
# Holdout ranking (faster, uses summary.json):
python scripts/batch_best_metrics.py \
--analysis-dir analysis_data/grid_search \
--metric holdout_auc_fused \
--acc-metric holdout_acc_fused \
--source summary \
--sort-by mean_auc --desc --top 10
The script assumes each run directory contains files named `fold{n}_epoch_log.csv`.
It reports runs that have all five folds (fold0..fold4) present by default.
"""
from __future__ import annotations
import argparse
import csv
import json
import math
import sys
import time
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Tuple
REQUIRED_FOLDS = {f"fold{i}_epoch_log.csv" for i in range(5)}
def to_float(value: Optional[object]) -> Optional[float]:
if value is None:
return None
if isinstance(value, (int, float)):
num = float(value)
if math.isnan(num):
return None
return num
if not isinstance(value, str):
return None
value = value.strip()
if not value:
return None
try:
num = float(value)
except ValueError:
return None
if math.isnan(num):
return None
return num
def best_value_from_csv(csv_path: Path, metric: str) -> Optional[Tuple[float, int]]:
best: Optional[Tuple[float, int]] = None
with csv_path.open("r", newline="") as fp:
reader = csv.DictReader(fp)
for row in reader:
val = to_float(row.get(metric))
if val is None:
continue
epoch = int(to_float(row.get("epoch")) or reader.line_num)
if best is None or val > best[0]:
best = (val, epoch)
return best
def render_progress(current: int, total: Optional[int], matched: int) -> str:
if total:
width = 30
filled = int(width * current / total)
bar = "#" * filled + "-" * (width - filled)
return f"[{bar}] {current}/{total} matched {matched}"
return f"Scanned {current} dirs, matched {matched}"
def find_run_directories(root: Path,
shallow: bool,
required_files: Iterable[str],
show_progress: bool) -> Iterable[Path]:
"""
Yield directories that look like HyperTower runs (contain at least the required fold logs).
"""
required_set = set(required_files)
if shallow:
entries = [entry for entry in root.iterdir() if entry.is_dir()]
entries.sort(key=lambda p: p.name)
total = len(entries)
matched = 0
last_update = 0.0
for idx, entry in enumerate(entries, start=1):
if show_progress:
now = time.monotonic()
if now - last_update >= 0.1 or idx == total:
msg = render_progress(idx, total, matched)
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
last_update = now
if not entry.is_dir():
continue
if all((entry / filename).is_file() for filename in required_set):
matched += 1
yield entry
if show_progress:
print(file=sys.stderr)
return
matched = 0
scanned = 0
last_update = 0.0
for dirpath, dirnames, filenames in os_walk_sorted(root):
scanned += 1
if show_progress:
now = time.monotonic()
if now - last_update >= 0.2:
msg = render_progress(scanned, None, matched)
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
last_update = now
files = set(filenames)
if required_set.issubset(files):
matched += 1
yield Path(dirpath)
if show_progress:
msg = render_progress(scanned, None, matched)
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
print(file=sys.stderr)
def os_walk_sorted(root: Path):
"""
Wrapper around os.walk that yields deterministic, sorted directory order.
"""
import os
for dirpath, dirnames, filenames in os.walk(root):
dirnames.sort()
filenames.sort()
yield dirpath, dirnames, filenames
def read_summary(run_dir: Path) -> Optional[Dict[str, object]]:
summary_path = run_dir / "summary.json"
if not summary_path.exists():
return None
try:
data = json.loads(summary_path.read_text())
except Exception:
return None
if not isinstance(data, dict):
return None
return data
def read_run_id(run_dir: Path, summary: Optional[Dict[str, object]] = None) -> str:
data = summary if summary is not None else read_summary(run_dir)
if data:
rid = data.get("run_id")
if isinstance(rid, str) and rid:
return rid
return run_dir.name
def mean(values: List[float]) -> Optional[float]:
return (sum(values) / len(values)) if values else None
def metric_from_stats(stats: Dict[str, object], metric: str) -> Optional[float]:
if stats.get("holdout_best_monitor") == metric:
best_val = to_float(stats.get("holdout_best_so_far"))
if best_val is not None:
return best_val
return to_float(stats.get(metric))
def task_from_summary(summary: Optional[Dict[str, object]]) -> Optional[str]:
if not summary:
return None
eval_mode = summary.get("eval_mode")
if isinstance(eval_mode, str):
mode = eval_mode.strip().lower()
if mode == "binary":
return "binary"
if mode in {"multiclass", "multi", "multi-class"}:
return "multiclass"
num_classes = summary.get("num_classes")
if isinstance(num_classes, (int, float)):
return "binary" if int(num_classes) <= 2 else "multiclass"
return None
def format_table(rows: List[Dict[str, Optional[object]]], columns: List[str]) -> str:
col_widths = {
col: max(len(col), max((len(fmt_value(row.get(col))) for row in rows), default=0))
for col in columns
}
header = " | ".join(col.ljust(col_widths[col]) for col in columns)
divider = "-+-".join("-" * col_widths[col] for col in columns)
body_lines = [
" | ".join(fmt_value(row.get(col)).ljust(col_widths[col]) for col in columns)
for row in rows
]
return "\n".join([header, divider, *body_lines])
def fmt_value(value: Optional[object]) -> str:
if value is None:
return ""
if isinstance(value, str):
return value
if isinstance(value, int):
return str(value)
return f"{value:.4f}"
def main() -> None:
ap = argparse.ArgumentParser(description="Aggregate best per-fold metrics from HyperTower runs.")
ap.add_argument("--analysis-dir", type=Path, default=Path("analysis_data"),
help="Directory containing run subdirectories (default: analysis_data)")
ap.add_argument("--metric", default="auc_fused",
help="Metric column to maximise (default: auc_fused)")
ap.add_argument("--acc-metric", default="acc_fused",
help="Accuracy column to maximise (default: acc_fused)")
ap.add_argument("--shallow", action="store_true",
help="Only scan directories directly under analysis-dir")
ap.add_argument("--source", choices=["epoch_logs", "summary"], default="epoch_logs",
help="Where to read metrics from (default: epoch_logs)")
ap.add_argument("--task", choices=["binary", "multiclass", "all"], default="all",
help="Filter runs by task type (default: all)")
ap.add_argument("--no-progress", action="store_true",
help="Disable progress output")
ap.add_argument("--match", default=None,
help="Only include run directories whose name contains this substring")
ap.add_argument("--sort-by", choices=["mean_auc", "mean_acc"], default=None,
help="Optional column to sort by (default: none)")
ap.add_argument("--desc", action="store_true",
help="Sort in descending order (default: ascending)")
ap.add_argument("--top", type=int, default=None,
help="Limit output to the top N rows after sorting")
ap.add_argument("--output-file", type=Path, default=None,
help="Optional path to write CSV summary")
args = ap.parse_args()
root = args.analysis_dir
if not root.exists():
raise SystemExit(f"Analysis directory not found: {root}")
rows: List[Dict[str, Optional[object]]] = []
missing_summary = 0
unknown_task = 0
required_files = REQUIRED_FOLDS if args.source == "epoch_logs" else ["summary.json"]
for run_dir in find_run_directories(
root,
shallow=args.shallow,
required_files=required_files,
show_progress=not args.no_progress,
):
if args.match and args.match not in run_dir.name:
continue
summary = None
task_label = None
if args.task != "all" or args.source == "summary":
summary = read_summary(run_dir)
if summary is None:
missing_summary += 1
continue
task_label = task_from_summary(summary)
if args.task != "all":
if task_label is None:
unknown_task += 1
continue
if task_label != args.task:
continue
run_id = read_run_id(run_dir, summary)
best_metrics: List[float] = []
best_accs: List[float] = []
if args.source == "summary":
folds = summary.get("fold_metrics") if summary else None
if not folds:
continue
for fold in folds:
stats = fold.get("stats") or {}
metric_val = metric_from_stats(stats, args.metric)
acc_val = metric_from_stats(stats, args.acc_metric)
if metric_val is None or acc_val is None:
best_metrics = []
best_accs = []
break
best_metrics.append(metric_val)
best_accs.append(acc_val)
else:
for fold_idx in range(5):
csv_path = run_dir / f"fold{fold_idx}_epoch_log.csv"
metric_entry = best_value_from_csv(csv_path, args.metric)
acc_entry = best_value_from_csv(csv_path, args.acc_metric)
if metric_entry is None or acc_entry is None:
# Skip this run if any fold is missing data
best_metrics = []
best_accs = []
break
best_metrics.append(metric_entry[0])
best_accs.append(acc_entry[0])
if not best_metrics or not best_accs:
continue
rows.append({
"run_id": run_id,
"task": task_label,
"relative_path": str(run_dir.relative_to(root)),
"mean_auc": mean(best_metrics),
"mean_acc": mean(best_accs),
})
if not rows:
print("No matching runs found.")
return
if args.sort_by:
def sort_key(row: Dict[str, Optional[float]]) -> float:
value = row.get(args.sort_by)
if value is None:
return float("-inf") if args.desc else float("inf")
return float(value)
rows.sort(key=sort_key, reverse=args.desc)
if args.top is not None:
rows = rows[:args.top]
columns = ["run_id", "task", "relative_path", "mean_auc", "mean_acc"]
if args.task != "all":
print(f"Task filter: {args.task}")
if args.match:
print(f"Name filter: {args.match}")
print(f"Runs: {len(rows)}\n")
print(format_table(rows, columns))
if args.output_file:
out_path = args.output_file
out_path.parent.mkdir(parents=True, exist_ok=True)
with out_path.open("w", newline="") as fp:
writer = csv.DictWriter(fp, fieldnames=columns)
writer.writeheader()
for row in rows:
writer.writerow(row)
print(f"\nSummary written to {out_path}")
if args.task != "all" and (missing_summary or unknown_task):
print(f"\nSkipped {missing_summary} runs without summary.json and {unknown_task} with unknown task type.")
if __name__ == "__main__":
main()
scripts/output_analysis/grid_search/plot_fold_auc_accuracy.py
Executable
+286
View File
@@ -0,0 +1,286 @@
#!/usr/bin/env python3
"""Aggregate per-fold metrics across runs and visualize AUC vs accuracy.
The script scans every `summary.json` under the provided analysis directory,
loads the per-fold macro AUC values, and combines them with per-fold
predictions to compute accuracy. Two scatter plots are produced:
1. AUC vs. fold index (with jitter) coloured by fold.
2. Accuracy (x-axis) vs. AUC (y-axis) coloured by fold.
This helps identify folds that persistently underperform across experiments.
"""
from __future__ import annotations
import argparse
import json
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterable, List, Optional
import numpy as np
try:
import matplotlib.pyplot as plt
from matplotlib.cm import get_cmap
from matplotlib.lines import Line2D
except ImportError as exc: # pragma: no cover - forward-friendly error for runtime
raise SystemExit("matplotlib is required to run this script") from exc
@dataclass
class FoldMetric:
run_id: str
fold: int
auc: float
accuracy: float
summary_path: Path
fusion_mode: Optional[str]
plot_head: str
HEAD_SUFFIX = {
"fused": "fused",
"metadata": "md",
"metadata_only": "md",
"image": "img",
"image_only": "img",
"img": "img",
"md": "md",
}
def infer_head(summary: Dict[str, object]) -> str:
"""Return the prediction head name used for evaluation."""
plot_head = summary.get("plot_head")
if isinstance(plot_head, str) and plot_head:
key = plot_head.lower()
if key in HEAD_SUFFIX:
return key
fusion_mode = summary.get("fusion_mode")
if isinstance(fusion_mode, str):
key = fusion_mode.lower()
if key in HEAD_SUFFIX:
return key
# Fall back to fused head if nothing else matches
return "fused"
def prediction_suffix(head: str) -> str:
key = head.lower()
if key in {"metadata", "metadata_only", "md"}:
return "md"
if key in {"image", "image_only", "img"}:
return "img"
return "fused"
def compute_accuracy(probs: np.ndarray, y_true: np.ndarray) -> float:
if probs.ndim == 1:
preds = (probs >= 0.5).astype(int)
else:
preds = np.argmax(probs, axis=1)
y_int = y_true.astype(int)
return float((preds == y_int).mean()) if y_int.size else np.nan
def load_summary(path: Path) -> Optional[Dict[str, object]]:
try:
with path.open("r") as f:
return json.load(f)
except Exception as exc:
print(f"[warn] Could not parse {path}: {exc}", file=sys.stderr)
return None
def collect_metrics(summary_path: Path) -> Iterable[FoldMetric]:
summary = load_summary(summary_path)
if not summary:
return []
# Only keep multiclass experiments (num_classes > 2 or eval_mode explicitly multiclass)
num_classes = summary.get("num_classes")
eval_mode = summary.get("eval_mode")
if (isinstance(num_classes, int) and num_classes <= 2) or (isinstance(eval_mode, str) and eval_mode.lower() == "binary"):
return []
head = infer_head(summary)
per_fold_auc = summary.get("per_fold_macro_ovr_auc") or summary.get("per_fold_auc")
if not isinstance(per_fold_auc, list):
# Fallback for summaries that only store fold_metrics[*].stats.
metric_key = f"auc_{prediction_suffix(head)}"
fold_metrics = summary.get("fold_metrics")
if not isinstance(fold_metrics, list):
return []
per_fold_auc = []
for entry in fold_metrics:
if not isinstance(entry, dict):
return []
stats = entry.get("stats")
if not isinstance(stats, dict):
return []
auc_val = stats.get(metric_key)
try:
per_fold_auc.append(float(auc_val))
except (TypeError, ValueError):
return []
suffix = prediction_suffix(head)
run_id = summary.get("run_id", summary_path.parent.name)
fusion_mode = summary.get("fusion_mode")
for fold_idx, auc_val in enumerate(per_fold_auc):
try:
auc = float(auc_val)
except (TypeError, ValueError):
continue
base = summary_path.parent
probs_path = base / f"fold{fold_idx}_probs_{suffix}.npy"
y_true_path = base / f"fold{fold_idx}_y_true.npy"
if not probs_path.exists() or not y_true_path.exists():
# fall back: if fused missing for metadata mode (or vice versa), try md or img
if suffix != "fused":
alt_probs_path = base / f"fold{fold_idx}_probs_fused.npy"
if alt_probs_path.exists():
probs_path = alt_probs_path
if not probs_path.exists():
print(
f"[warn] Missing predictions for fold {fold_idx} in {base}; skipped",
file=sys.stderr,
)
continue
try:
probs = np.load(probs_path)
y_true = np.load(y_true_path)
except Exception as exc:
print(f"[warn] Failed loading predictions for {base}: {exc}", file=sys.stderr)
continue
accuracy = compute_accuracy(probs, y_true)
yield FoldMetric(
run_id=str(run_id),
fold=fold_idx,
auc=auc,
accuracy=accuracy,
summary_path=summary_path,
fusion_mode=fusion_mode if isinstance(fusion_mode, str) else None,
plot_head=head,
)
def build_plot(metrics: List[FoldMetric], output: Path, jitter: float, seed: int, show: bool) -> None:
rng = np.random.default_rng(seed)
folds = sorted({m.fold for m in metrics})
fold_to_color: Dict[int, tuple] = {}
cmap = get_cmap("tab10", max(len(folds), 1))
for idx, fold in enumerate(folds):
fold_to_color[fold] = cmap(idx)
# Prepare arrays for plotting
aucs = np.array([m.auc for m in metrics])
accs = np.array([m.accuracy for m in metrics])
fold_indices = np.array([m.fold for m in metrics])
colors = [fold_to_color[m.fold] for m in metrics]
jitter_offsets = rng.uniform(-jitter, jitter, size=len(metrics))
fig, axes = plt.subplots(1, 2, figsize=(13, 5), constrained_layout=True)
# Panel 1: Fold vs AUC scatter with jitter
ax0 = axes[0]
ax0.scatter(fold_indices + 1 + jitter_offsets, aucs, c=colors, edgecolor="k", linewidth=0.4, alpha=0.85)
ax0.set_xticks([f + 1 for f in folds])
ax0.set_xlabel("Fold index")
ax0.set_ylabel("Macro AUC")
ax0.set_title("Per-fold AUC across runs")
ax0.grid(True, linestyle=":", linewidth=0.5, alpha=0.4)
# Panel 2: Accuracy vs AUC scatter
ax1 = axes[1]
ax1.scatter(accs, aucs, c=colors, edgecolor="k", linewidth=0.4, alpha=0.85)
ax1.set_xlabel("Accuracy")
ax1.set_ylabel("Macro AUC")
ax1.set_title("Accuracy vs AUC by fold")
ax1.grid(True, linestyle=":", linewidth=0.5, alpha=0.4)
# Shared legend
legend_handles = [
Line2D(
[0],
[0],
marker="o",
color="w",
label=f"Fold {fold + 1}",
markerfacecolor=fold_to_color[fold],
markeredgecolor="k",
markersize=8,
)
for fold in folds
]
for ax in axes:
ax.legend(handles=legend_handles, frameon=False, loc="lower right")
fig.suptitle("Fold-level performance across experiments", fontsize=14)
output.parent.mkdir(parents=True, exist_ok=True)
fig.savefig(output, dpi=200)
print(f"Saved plot to {output}")
if show:
plt.show()
plt.close(fig)
def print_summary(metrics: List[FoldMetric]) -> None:
total_runs = len({m.run_id for m in metrics})
print(f"Collected {len(metrics)} fold metrics from {total_runs} runs.")
by_fold: Dict[int, List[FoldMetric]] = {}
for metric in metrics:
by_fold.setdefault(metric.fold, []).append(metric)
for fold, entries in sorted(by_fold.items()):
aucs = np.array([m.auc for m in entries])
accs = np.array([m.accuracy for m in entries])
print(
f" Fold {fold + 1}: AUC {aucs.mean():.3f} ± {aucs.std(ddof=0):.3f} | "
f"Accuracy {accs.mean():.3f} ± {accs.std(ddof=0):.3f} (n={len(entries)})"
)
def main(argv: Optional[List[str]] = None) -> int:
parser = argparse.ArgumentParser(description="Plot per-fold AUCs and accuracies across runs.")
parser.add_argument(
"--analysis-root",
default="analysis_data",
help="Root directory that contains run folders with summary.json files (default: analysis_data)",
)
parser.add_argument(
"--output",
default="analysis_data/fold_auc_vs_accuracy.png",
help="Where to save the generated figure (default: analysis_data/fold_auc_vs_accuracy.png)",
)
parser.add_argument("--jitter", type=float, default=0.08, help="Horizontal jitter for fold scatter plot")
parser.add_argument("--seed", type=int, default=17, help="Random seed for jitter replication")
parser.add_argument("--show", action="store_true", help="Display the plot interactively after saving")
args = parser.parse_args(argv)
analysis_root = Path(args.analysis_root)
if not analysis_root.exists():
raise SystemExit(f"Analysis root {analysis_root} does not exist")
summary_files = sorted(analysis_root.rglob("summary.json"))
if not summary_files:
raise SystemExit(f"No summary.json files found under {analysis_root}")
metrics: List[FoldMetric] = []
for summary_path in summary_files:
metrics.extend(collect_metrics(summary_path))
if not metrics:
raise SystemExit("No fold metrics collected. Check that prediction files are present.")
print_summary(metrics)
build_plot(metrics, Path(args.output), jitter=args.jitter, seed=args.seed, show=args.show)
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/output_analysis/segmenter/filter_low_dice.py
Executable
+78
View File
@@ -0,0 +1,78 @@
"""Filter segmentation metrics rows with near-zero Dice scores."""
from __future__ import annotations
import argparse
from pathlib import Path
import pandas as pd
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description=(
"Drop samples where both disc and cup Dice are below a threshold "
"(default 0.01) and report how many were removed."
)
)
parser.add_argument("input", type=Path, help="Path to metrics CSV to filter")
parser.add_argument(
"--output",
type=Path,
help="Destination CSV. Defaults to <input stem>_filtered.csv in the same directory.",
)
parser.add_argument(
"--threshold",
type=float,
default=0.01,
help="Dice cutoff; rows with both dice_disc and dice_cup below this are removed.",
)
parser.add_argument(
"--keep-summary",
action="store_true",
help="Always keep summary rows (sample_id == '__mean__').",
)
return parser.parse_args()
def main() -> None:
args = parse_args()
df = pd.read_csv(args.input)
mask_low = (df["dice_disc"] < args.threshold) & (df["dice_cup"] < args.threshold)
if args.keep_summary and "sample_id" in df.columns:
mask_low &= df["sample_id"].ne("__mean__")
removed = int(mask_low.sum())
filtered = df.loc[~mask_low].copy()
# Recompute summary if original file contained one
if "sample_id" in filtered.columns:
summary_mask = filtered["sample_id"].eq("__mean__")
filtered = filtered.loc[~summary_mask].copy()
if not filtered.empty:
summary = filtered[["dice_disc", "dice_cup"]].mean()
summary_row = {
"sample_id": "__mean__",
"dataset": "summary",
"split": "summary",
"dice_disc": summary["dice_disc"],
"dice_cup": summary["dice_cup"],
}
filtered = pd.concat([filtered, pd.DataFrame([summary_row])], ignore_index=True)
remaining = len(filtered)
output_path = args.output
if output_path is None:
output_path = args.input.with_name(f"{args.input.stem}_filtered.csv")
filtered.to_csv(output_path, index=False)
print(f"Removed rows: {removed}")
print(f"Remaining rows: {remaining}")
print(f"Filtered metrics saved to: {output_path}")
if __name__ == "__main__":
main()
scripts/output_analysis/visualizations/aggregate_roc_perclass_all_models.py
Executable
+241
View File
@@ -0,0 +1,241 @@
#!/usr/bin/env python3
"""
Per-class ROC: one figure per class (multiclass) OR one figure total (binary),
with ALL models (runs under a tag) plotted as separate lines.
Outputs under analysis_data/:
- multiclass:
<tag>_class0_roc.png (e.g., Healthy)
<tag>_class1_roc.png (e.g., Glaucoma)
<tag>_class2_roc.png (e.g., Suspect)
<tag>_perclass_summary.json
- binary:
<tag>_binary_roc.png
<tag>_perclass_summary.json
"""
import argparse, json, re
from pathlib import Path
import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, auc, roc_auc_score
HEAD_ALIASES = {"image": ["image","img"], "fused": ["fused"], "metadata": ["metadata","md"]}
def find_run_dirs(tag_prefix: str, analysis_dir: Path):
return sorted([p for p in analysis_dir.glob(f"{tag_prefix}_*") if p.is_dir()])
def read_summary(run_dir: Path) -> dict:
p = run_dir / "summary.json"
if p.exists():
try:
return json.loads(p.read_text())
except Exception:
pass
return {}
def find_folds(run_dir: Path, head: str):
variants = HEAD_ALIASES.get(head, [head])
y_files = sorted(run_dir.glob("fold*_y_true.npy"))
folds = []
for yf in y_files:
m = re.search(r"fold(\d+)_y_true\.npy$", yf.name)
if not m: continue
idx = int(m.group(1))
if any((run_dir / f"fold{idx}_probs_{v}.npy").exists() for v in variants):
folds.append(idx)
return folds
def load_probs(run_dir: Path, fold: int, head: str):
variants = HEAD_ALIASES.get(head, [head])
y = np.load(run_dir / f"fold{fold}_y_true.npy")
p = None
tried = []
for v in variants:
pp = run_dir / f"fold{fold}_probs_{v}.npy"
tried.append(pp.name)
if pp.exists():
p = np.load(pp); break
if p is None:
raise FileNotFoundError(f"Missing probs for fold {fold} in {run_dir}; tried {tried}")
return y, p
def infer_mode_from_files(run_dir: Path, head: str):
f = find_folds(run_dir, head)
if not f: return None
_, p = load_probs(run_dir, f[0], head)
if p.ndim == 2 and p.shape[1] == 2: return "binary"
if p.ndim == 2 and p.shape[1] >= 3: return "multiclass"
return None
def per_class_roc(y, p):
"""Return {k: (fpr, tpr, auc)} for OVR."""
K = p.shape[1]
out = {}
for k in range(K):
yb = (y == k).astype(np.uint8)
fpr, tpr, _ = roc_curve(yb, p[:, k])
out[k] = (fpr, tpr, auc(fpr, tpr) if len(fpr) > 1 else np.nan)
return out
def make_per_model_class_curves(run_dir: Path, head: str, mode: str):
"""
Returns:
label (model/backbone name),
class_curves: dict[k] -> dict with keys:
'fpr': grid, 'tpr_mean': mean across folds on grid, 'auc_mean': mean across folds,
'tpr_std' and 'auc_std' also included.
K = number of classes (2 or 3+)
"""
summary = read_summary(run_dir)
label = summary.get("backbone") or run_dir.name
folds = find_folds(run_dir, head)
if not folds:
return None
# collect per-fold per-class curves
per_fold = []
for f in folds:
y, p = load_probs(run_dir, f, head)
if mode == "binary":
keep = np.isin(y, [0,1])
if keep.sum() == 0:
continue
y, p = y[keep], p[keep]
if p.shape[1] > 2: # safety; binary should have 2 cols
p = p[:, :2]
else:
if p.ndim != 2 or p.shape[1] < 3:
continue
per_fold.append(per_class_roc(y, p))
if not per_fold:
return None
# interpolate on a common grid, avg across folds
grid = np.linspace(0, 1, 501)
K = max(per_fold[0].keys()) + 1
class_curves = {}
for k in range(K):
tprs, aucs = [], []
for d in per_fold:
if k not in d:
continue
fpr, tpr, a = d[k]
tprs.append(np.interp(grid, fpr, tpr))
aucs.append(a)
if not tprs:
continue
tprs = np.vstack(tprs)
class_curves[k] = {
"fpr": grid,
"tpr_mean": tprs.mean(axis=0),
"tpr_std": tprs.std(axis=0),
"auc_mean": float(np.nanmean(aucs)),
"auc_std": float(np.nanstd(aucs)),
}
return label, class_curves
def main():
ap = argparse.ArgumentParser(description="Per-class ROC with all models as separate lines.")
ap.add_argument("--tag", required=True, help="analysis_data prefix like 'papergrid'")
ap.add_argument("--head", default="image", choices=["image","fused","metadata"])
ap.add_argument("--mode", choices=["binary","multiclass"], required=True,
help="Select which experiment style to aggregate.")
ap.add_argument("--fusion-mode", choices=["image_only","fused","metadata_only","vote"], default=None,
help="Filter runs by fusion mode to avoid mixing.")
ap.add_argument("--analysis-dir", default="analysis_data")
ap.add_argument("--class-names", nargs="*", default=["Healthy","Glaucoma","Suspect"])
ap.add_argument("--shade", action="store_true", help="Shade ±1 SD per model (can get busy).")
args = ap.parse_args()
analysis_dir = Path(args.analysis_dir) / args.tag
run_dirs_all = find_run_dirs(args.tag, analysis_dir)
if not run_dirs_all:
raise SystemExit(f"No run directories found starting with '{args.tag}_' under {analysis_dir}")
# filter runs
selected = []
skipped = []
for rd in run_dirs_all:
sj = read_summary(rd)
m = sj.get("eval_mode") or infer_mode_from_files(rd, args.head)
if m != args.mode:
skipped.append((rd, f"mode={m}")); continue
if args.fusion_mode:
fm = sj.get("fusion_mode")
if fm and fm != args.fusion_mode:
skipped.append((rd, f"fusion_mode={fm}")); continue
selected.append(rd)
if not selected:
raise SystemExit("No runs matched filters (mode/fusion-mode).")
# build per-model curves
per_model = [] # list of (label, class_curves)
for rd in selected:
res = make_per_model_class_curves(rd, args.head, args.mode)
if res is None:
skipped.append((rd, "no_usable_folds")); continue
per_model.append(res)
if not per_model:
raise SystemExit("No usable runs after fold parsing/interpolation.")
# determine classes to plot
maxK = max((max(curves.keys())+1) for _, curves in per_model)
if args.mode == "binary":
# Only class 1 (positive) is typically plotted
classes_to_plot = [1]
class_names = [args.class_names[1] if len(args.class_names) > 1 else "Positive"]
outfile_names = [f"{args.tag}_binary_roc.png"]
title_suffixes = ["Binary (positive class)"]
else:
classes_to_plot = list(range(min(3, maxK))) # usually 0,1,2
class_names = [args.class_names[i] if i < len(args.class_names) else f"class {i}" for i in classes_to_plot]
outfile_names = [f"{args.tag}_class{i}_roc.png" for i in classes_to_plot]
title_suffixes = [f"Class: {name}" for name in class_names]
# plot per class: all models on same axes
out_json = {"tag": args.tag, "mode": args.mode, "head": args.head,
"fusion_mode_filter": args.fusion_mode, "figures": []}
for k, cname, out_name, t_suffix in zip(classes_to_plot, class_names, outfile_names, title_suffixes):
fig = plt.figure(figsize=(10, 8)); ax = fig.add_subplot(111)
ax.plot([0,1],[0,1], linestyle="--", linewidth=1)
ax.set_xlabel("False Positive Rate"); ax.set_ylabel("True Positive Rate")
title_bits = [f"Combined ROC — {args.tag}", t_suffix, f"[{args.head}]"]
if args.fusion_mode: title_bits.append(f"[{args.fusion_mode}]")
ax.set_title(" — ".join(title_bits))
entries = []
for label, curves in per_model:
if k not in curves:
continue
c = curves[k]
ax.plot(c["fpr"], c["tpr_mean"], linewidth=2,
label=f"{label} (AUC {c['auc_mean']:.3f}±{c['auc_std']:.3f})")
if args.shade:
ax.fill_between(c["fpr"],
np.maximum(c["tpr_mean"] - c["tpr_std"], 0),
np.minimum(c["tpr_mean"] + c["tpr_std"], 1),
alpha=0.10)
entries.append({"label": label, "auc_mean": c["auc_mean"], "auc_std": c["auc_std"]})
ax.legend(loc="lower right")
fig.tight_layout()
out_path = analysis_dir / out_name
fig.savefig(out_path, dpi=160); plt.close(fig)
out_json["figures"].append({
"class_index": k, "class_name": cname, "output_png": str(out_path),
"models": entries
})
# metadata file
meta_path = analysis_dir / f"{args.tag}_perclass_summary.json"
meta_path.write_text(json.dumps(out_json, indent=2), encoding="utf-8")
print(f"Wrote figures + {meta_path}")
if __name__ == "__main__":
main()
scripts/output_analysis/visualizations/rebuild_run_best_plots.py
Executable
+447
View File
@@ -0,0 +1,447 @@
#!/usr/bin/env python3
"""
Recompute per-fold ROC plots for a completed multifold run using the saved
best checkpoints instead of the final epoch.
Example:
python scripts/rebuild_run_best_plots.py \
--run-dir analysis_data/1029_Baseline_Balanced_Resnet/1029_Baseline_Balanced_Resnet_20251029_163906 \
--head image
"""
import argparse
import logging
import json
from pathlib import Path
from types import SimpleNamespace
import matplotlib
import sys
REPO_ROOT = Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
import numpy as np # noqa: E402
import pandas as pd # noqa: E402
import torch # noqa: E402
from sklearn.metrics import auc, roc_auc_score, roc_curve # noqa: E402
from classes import build_papila_clinical # noqa: E402
from classes.hypertower import HyperTower # noqa: E402
try: # Allow checkpoints that stored pandas DataFrames in their args.
from torch.serialization import add_safe_globals # type: ignore
add_safe_globals([pd.DataFrame])
except (ImportError, AttributeError):
pass
def parse_args() -> argparse.Namespace:
ap = argparse.ArgumentParser(description="Rebuild ROC plots for an existing multifold run.")
ap.add_argument("--run-dir", required=True, type=Path, help="Path to the run directory under analysis_data.")
ap.add_argument("--head", default="image", choices=["image", "fused", "metadata"], help="Which prediction head to plot.")
ap.add_argument("--class-names", nargs="*", default=None, help="Optional class names to control plot labels.")
ap.add_argument("--overwrite", action="store_true", help="Overwrite existing .npy probability dumps if present.")
ap.add_argument(
"--use-holdout",
action="store_true",
help="Evaluate checkpoints on the saved holdout set instead of the fold validation splits.",
)
return ap.parse_args()
def load_cli_args(run_dir: Path) -> dict:
cli_path = run_dir / "cli_args.json"
if not cli_path.exists():
raise FileNotFoundError(f"Missing cli_args.json in {run_dir}")
with cli_path.open("r", encoding="utf-8") as fh:
return json.load(fh)
def load_summary(run_dir: Path) -> dict:
summary_path = run_dir / "summary.json"
if not summary_path.exists():
raise FileNotFoundError(f"Missing summary.json in {run_dir}")
with summary_path.open("r", encoding="utf-8") as fh:
return json.load(fh)
def prepare_clinical(cli_args: dict, run_dir: Path) -> tuple:
clinical = build_papila_clinical(
cli_args["image_dir"],
cli_args["clinical_dir"],
cli_args["label_col"],
cli_args["cat_cols"],
n_splits=cli_args["n_splits"],
random_seed=cli_args["fold_seed"],
)
holdout_path = run_dir / "holdout.csv"
holdout_df = pd.read_csv(holdout_path) if holdout_path.exists() else None
if holdout_df is not None:
if cli_args["eval_mode"] == "binary":
holdout_df = holdout_df[holdout_df[cli_args["label_col"]].isin([0, 1])].reset_index(drop=True)
join_cols = [c for c in holdout_df.columns if c in clinical.df.columns]
if not join_cols:
raise RuntimeError("Holdout CSV found but no overlapping columns with clinical dataframe.")
marker = holdout_df.assign(_holdout_marker=1)
merged = clinical.df.merge(marker, on=join_cols, how="left")
train_df = merged[merged["_holdout_marker"].isna()].drop(columns=["_holdout_marker"]).reset_index(drop=True)
clinical.frames = [train_df.copy()]
clinical.df = train_df.copy()
clinical._infer_or_validate_feature_types()
clinical._compute_numeric_stats()
clinical._build_cat_maps()
clinical._compute_feature_dim()
clinical._build_kfold_indices()
return clinical, holdout_df
def build_ht_args(cli_args: dict, fold: int, run_dir: Path, models_dir: Path, holdout_df):
# Copy of the training-time namespace so HyperTower can be re-instantiated.
return SimpleNamespace(
image_dir=cli_args["image_dir"],
clinical_dir=cli_args["clinical_dir"],
label_col=cli_args["label_col"],
cat_cols=cli_args["cat_cols"],
batch_size=cli_args["batch_size"],
epochs=cli_args["epochs"],
lr=cli_args["lr"],
num_classes=cli_args["num_classes"],
img_augment=cli_args.get("img_augment", True),
focal_gamma=cli_args.get("focal_gamma", 0.0),
eval_mode=cli_args["eval_mode"],
fold=fold,
run_dir=str(run_dir),
models_dir=str(models_dir),
backbone=cli_args["backbone"],
freeze_ratio=cli_args["freeze_ratio"],
fusion_mode=cli_args["fusion_mode"],
use_se=cli_args.get("use_se", True),
se_reduction=cli_args.get("se_reduction", 16),
se_pre_norm=cli_args.get("se_pre_norm", True),
se_where=cli_args.get("se_where", "bridge"),
se_reduction_tower=cli_args.get("se_reduction_tower", cli_args.get("se_reduction", 16)),
se_pre_norm_tower=cli_args.get("se_pre_norm_tower", cli_args.get("se_pre_norm", True)),
warmup_tower_epochs=cli_args.get("warmup_tower_epochs", 0),
warmup_fused_epochs=cli_args.get("warmup_fused_epochs", 0),
gradual_thaw=cli_args.get("gradual_thaw", False),
thaw_phase_duration=cli_args.get("thaw_phase_duration", 5),
thaw_ratio=cli_args.get("thaw_ratio", 0.33),
thaw_target=cli_args.get("thaw_target", "image"),
thaw_start_epoch=cli_args.get("thaw_start_epoch", -1),
initial_freeze=cli_args.get("initial_freeze", False),
bcd_prob=0.5,
bcd_p0=0.20,
bcd_min=0.05,
bcd_max=0.30,
bcd_k=0.4,
bcd_metric="auc",
bcd_alpha_batch=0.2,
bcd_alpha_tower=0.3,
bcd_explore_floor=0.15,
aux_img=0.05,
aux_md=0.05,
aux_detach=True,
ema_alpha=0.9,
entropy_ema=0.7,
early_stop=cli_args.get("early_stop", False),
early_metric=cli_args.get("early_metric"),
early_mode=cli_args.get("early_mode", "auto"),
early_patience=cli_args.get("early_patience", 7),
early_min_delta=cli_args.get("early_min_delta", 0.0),
checkpoint_best=cli_args.get("checkpoint_best", False),
holdout_df=holdout_df,
img_crop_manifest=cli_args.get("img_crop_manifest"),
img_crop_weights=cli_args.get("img_crop_weights"),
img_crop_normalize=cli_args.get("img_crop_normalize"),
img_crop_threshold=cli_args.get("img_crop_threshold"),
img_crop_scale=cli_args.get("img_crop_scale", 2.5),
img_crop_size=cli_args.get("img_crop_size", 224),
img_crop_cache=cli_args.get("img_crop_cache"),
img_crop_tta=cli_args.get("img_crop_tta", False),
img_crop_gt=cli_args.get("img_crop_gt", False),
img_geometry_features=cli_args.get("img_geometry_features", False),
balanced_sampler=cli_args.get("balanced_sampler", False),
)
def collect_logits(ht, loader):
"""Mirror Multifold.eval_collect_logits but for a provided loader."""
device = ht.device
ht.img_tower.eval()
ht.md_tower.eval()
outputs = []
with torch.no_grad():
if ht.mode == "vote":
ht.head_img.eval()
ht.head_md.eval()
ht.vote.eval()
else:
ht.bridge.eval()
for batch in loader:
if len(batch) == 4:
imgs, metas, geometry, labels = batch
else:
imgs, metas, labels = batch
geometry = None
imgs = imgs.to(device)
metas = metas.to(device)
labels = labels.to(device)
if geometry is not None and geometry.numel() > 0:
geometry = geometry.to(device)
else:
geometry = None
if ht.mode == "vote":
img_feats = ht.img_tower(imgs, geometry)
md_feats = ht.md_tower(metas)
out_img = ht.head_img(img_feats)
out_md = ht.head_md(md_feats)
out_fused = ht.vote(out_img, out_md)
else:
img_feats = ht.img_tower(imgs, geometry)
md_feats = ht.md_tower(metas)
result = ht.bridge(img_feats, md_feats)
if isinstance(result, tuple):
out_fused, out_img, out_md = result
else:
out_fused, out_img, out_md = result, None, None
outputs.append(
(
labels.detach().cpu().numpy(),
torch.softmax(out_fused, dim=1).detach().cpu().numpy() if out_fused is not None else None,
torch.softmax(out_img, dim=1).detach().cpu().numpy() if out_img is not None else None,
torch.softmax(out_md, dim=1).detach().cpu().numpy() if out_md is not None else None,
)
)
if not outputs:
return np.array([]), None, None, None
y_all, pf, pi, pm = zip(*outputs)
y_true = np.concatenate(y_all, axis=0)
probs_f = np.concatenate([p for p in pf if p is not None], axis=0) if any(p is not None for p in pf) else None
probs_i = np.concatenate([p for p in pi if p is not None], axis=0) if any(p is not None for p in pi) else None
probs_m = np.concatenate([p for p in pm if p is not None], axis=0) if any(p is not None for p in pm) else None
return y_true, probs_f, probs_i, probs_m
def compute_per_class_curves(y_true, probs):
if probs is None:
return {}
num_classes = probs.shape[1]
curves = {}
for k in range(num_classes):
y_bin = (y_true == k).astype(np.uint8)
fpr, tpr, _ = roc_curve(y_bin, probs[:, k])
curves[k] = {"fpr": fpr, "tpr": tpr, "auc": auc(fpr, tpr) if len(fpr) > 1 else np.nan}
return curves
def choose_head_probs(head: str, probs_f, probs_i, probs_m):
if head == "fused":
return probs_f
if head == "metadata":
return probs_m
return probs_i
def ensure_binary_slice(y_true, *arrays):
mask = np.isin(y_true, [0, 1])
filtered = [y_true[mask]]
for arr in arrays:
if arr is None:
filtered.append(None)
else:
filtered.append(arr[mask])
return filtered
def plot_overlays(per_fold_curves, out_dir: Path, class_names: list[str], head: str, suffix: str = ""):
keys = sorted({k for _, curves in per_fold_curves for k in curves.keys()})
if not keys:
return
name_map = {k: (class_names[k] if k < len(class_names) else f"class_{k}") for k in keys}
out_dir.mkdir(parents=True, exist_ok=True)
for k in keys:
fig = plt.figure(figsize=(10, 8))
ax = fig.add_subplot(111)
ax.plot([0, 1], [0, 1], linestyle="--", linewidth=1, color="grey")
for fold_idx, curves in per_fold_curves:
if k not in curves:
continue
fpr = curves[k]["fpr"]
tpr = curves[k]["tpr"]
auc_val = curves[k]["auc"]
label = f"Fold {fold_idx} (AUC {auc_val:.3f})" if auc_val == auc_val else f"Fold {fold_idx}"
ax.plot(fpr, tpr, linewidth=1.5, label=label)
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(f"{head} head — {name_map[k]} ROC per fold")
ax.legend(loc="lower right")
fig.tight_layout()
safe_name = name_map[k].replace(" ", "_")
suffix_str = suffix if suffix else ""
fig.savefig(out_dir / f"roc_{head}_{safe_name}_perfold{suffix_str}.png", dpi=160)
plt.close(fig)
def plot_mean_sd(per_fold_curves, out_dir: Path, class_names: list[str], head: str, suffix: str = ""):
keys = sorted({k for _, curves in per_fold_curves for k in curves.keys()})
if not keys:
return
grid = np.linspace(0, 1, 501)
fig = plt.figure(figsize=(10, 8))
ax = fig.add_subplot(111)
ax.plot([0, 1], [0, 1], linestyle="--", linewidth=1, color="grey")
for k in keys:
tprs = []
aucs = []
for _, curves in per_fold_curves:
if k not in curves:
continue
fpr = curves[k]["fpr"]
tpr = curves[k]["tpr"]
aucs.append(curves[k]["auc"])
tprs.append(np.interp(grid, fpr, tpr))
if not tprs:
continue
tprs = np.vstack(tprs)
mean = tprs.mean(axis=0)
std = tprs.std(axis=0)
label = class_names[k] if k < len(class_names) else f"class_{k}"
label = f"{label} (AUC {np.nanmean(aucs):.3f}±{np.nanstd(aucs):.3f})"
ax.plot(grid, mean, linewidth=2, label=label)
ax.fill_between(grid, np.maximum(mean - std, 0), np.minimum(mean + std, 1), alpha=0.15)
ax.set_xlabel("False Positive Rate")
ax.set_ylabel("True Positive Rate")
ax.set_title(f"Mean OVR ROC (±1 SD) — {head} head")
ax.legend(loc="lower right")
fig.tight_layout()
suffix_str = suffix if suffix else ""
out_dir.mkdir(parents=True, exist_ok=True)
fig.savefig(out_dir / f"roc_{head}_mean_ovr{suffix_str}.png", dpi=160)
plt.close(fig)
def main():
args = parse_args()
run_dir = args.run_dir.resolve()
cli_args = load_cli_args(run_dir)
summary = load_summary(run_dir)
class_names = (
args.class_names
if args.class_names
else (cli_args.get("class_names") or (["Healthy", "Glaucoma"] if cli_args["eval_mode"] == "binary" else ["Healthy", "Glaucoma", "Suspect"]))
)
shortname = cli_args.get("shortname") or run_dir.parent.name
run_id = cli_args.get("run_id") or run_dir.name
base_models_dir = Path("models") / shortname / run_id
clinical, holdout_df = prepare_clinical(cli_args, run_dir)
if args.use_holdout and holdout_df is None:
raise SystemExit("Holdout metrics requested but no holdout.csv found for this run.")
per_fold_curves = []
fold_aucs = []
head = args.head
file_suffix = "_holdout" if args.use_holdout else ""
for fold_entry in summary.get("fold_metrics", []):
fold_idx = int(fold_entry["fold"])
best_epoch = fold_entry.get("best_epoch")
if not best_epoch:
print(f"[skip] Fold {fold_idx}: no best_epoch recorded.")
continue
fold_models_dir = base_models_dir / f"fold{fold_idx}"
best_checkpoint = fold_models_dir / "model_best.pt"
if not best_checkpoint.exists():
print(f"[warning] Fold {fold_idx}: missing model_best.pt at {best_checkpoint}")
continue
ht_args = build_ht_args(cli_args, fold_idx, run_dir, fold_models_dir, holdout_df)
ht = HyperTower(clinical, ht_args)
for handler in list(ht.logger.handlers):
handler.close()
ht.logger.handlers = [logging.NullHandler()]
train_log_path = Path("train.log")
if train_log_path.exists() and train_log_path.stat().st_size == 0:
try:
train_log_path.unlink()
except OSError:
pass
try:
state = torch.load(best_checkpoint, map_location=ht.device, weights_only=False)
except TypeError:
state = torch.load(best_checkpoint, map_location=ht.device)
ht._restore_from_state(state)
if args.use_holdout:
eval_df = holdout_df.copy()
else:
_, eval_df = clinical.get_split_dfs(fold_idx)
if cli_args["eval_mode"] == "binary":
eval_df = eval_df[eval_df[cli_args["label_col"]].isin([0, 1])].reset_index(drop=True)
if eval_df.empty:
print(f"[warning] Fold {fold_idx}: evaluation dataframe is empty; skipping.")
continue
ht.test_loader = ht._make_loader_for_df(eval_df, is_train=False)
y_true, probs_f, probs_i, probs_m = collect_logits(ht, ht.test_loader)
if cli_args["eval_mode"] == "binary":
y_true, probs_f, probs_i, probs_m = ensure_binary_slice(y_true, probs_f, probs_i, probs_m)
head_probs = choose_head_probs(head, probs_f, probs_i, probs_m)
if head_probs is None:
print(f"[skip] Fold {fold_idx}: head '{head}' not available.")
continue
if head_probs.shape[1] >= 2:
head_probs = head_probs[:, :2]
if args.overwrite:
base = run_dir / f"fold{fold_idx}{file_suffix}"
np.save(f"{base}_y_true.npy", y_true)
if probs_f is not None:
np.save(f"{base}_probs_fused.npy", probs_f)
if probs_i is not None:
np.save(f"{base}_probs_img.npy", probs_i)
if probs_m is not None:
np.save(f"{base}_probs_md.npy", probs_m)
curves = compute_per_class_curves(y_true, head_probs)
per_fold_curves.append((fold_idx, curves))
try:
if head_probs.shape[1] > 2:
fold_auc = roc_auc_score(y_true, head_probs, multi_class="ovr", average="macro")
else:
target_scores = head_probs[:, 1] if head_probs.shape[1] > 1 else head_probs[:, 0]
fold_auc = roc_auc_score(y_true, target_scores)
fold_aucs.append(fold_auc)
print(f"[info] Fold {fold_idx}: best epoch {best_epoch}, AUC={fold_auc:.4f}")
except Exception:
print(f"[warning] Fold {fold_idx}: unable to compute AUC.")
if not per_fold_curves:
raise SystemExit("No folds processed; nothing to plot.")
plots_dir = run_dir / "plots"
plot_overlays(per_fold_curves, plots_dir, class_names, head, file_suffix)
plot_mean_sd(per_fold_curves, plots_dir, class_names, head, file_suffix)
if fold_aucs:
print(f"[info] {head} head mean AUC across folds: {np.mean(fold_aucs):.4f} ± {np.std(fold_aucs):.4f}")
print(f"Plots regenerated under {plots_dir}")
if __name__ == "__main__":
main()
scripts/testing/run_multifold_smoketest.sh
Executable
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env bash
# Quick smoke-test for run_multifold: runs two 1-epoch configs.
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT_DIR"
MANIFEST="manifest.csv"
IMAGENET_WEIGHTS="models/unet_segmenter/norm_imagenet/best.pt"
if [[ ! -f "$MANIFEST" ]]; then
echo "Missing $MANIFEST; run scripts/main/refuge/build_manifest.py first." >&2
exit 1
fi
if [[ ! -f "$IMAGENET_WEIGHTS" ]]; then
echo "Missing $IMAGENET_WEIGHTS; train the imagenet-normalized UNet first." >&2
exit 1
fi
COMMON_ARGS=(
--backbone resnet50
--fusion-mode fused
--epochs 1
--batch-size 4
--img-crop-manifest "$MANIFEST"
--img-crop-weights "$IMAGENET_WEIGHTS"
--img-crop-normalize imagenet
--shortname smoketest
--holdout-per-class 12
)
echo "[smoketest] Binary eval, fused head"
python scripts/run_multifold.py \
"${COMMON_ARGS[@]}" \
--eval_mode binary \
--run-id smoketest_binary
echo "→ Results under analysis_data/smoketest/smoketest_binary"
echo "[smoketest] Multiclass eval, fused head"
python scripts/run_multifold.py \
"${COMMON_ARGS[@]}" \
--eval_mode multiclass \
--run-id smoketest_multiclass
echo "→ Results under analysis_data/smoketest/smoketest_multiclass"
scripts/testing/test_v1_v2_config_compare.py
+798
View File
@@ -0,0 +1,798 @@
from __future__ import annotations
import argparse
from pathlib import Path
from types import SimpleNamespace
from typing import Any, Dict, List, Optional
import sys
import random
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
import numpy as np
import torch
from torch import nn
from classes.frontend import Multifold
from classes.bridge import Bridge, VoteBridge
from classes.dataset import ClinicalDataset
from classes.hypertower import _ClinicalView
from classes.image_tower import ImageTower
from classes.md_tower import MDTower
from classes.papila_builders import build_papila_clinical
from classes.v2 import (
PatientSplit,
SlotLoaderFactory,
SlotDataset,
slot_collate,
assemble_config,
build_model_bundle,
build_papila_profile,
resolve_imports,
)
from classes.v2.split_manager import PatientFirstSplitManager
def build_v1_defaults() -> Dict[str, Any]:
parser = Multifold.build_parser()
args = parser.parse_args([])
clinical = build_papila_clinical(
image_dir=args.image_dir,
clinical_dir=args.clinical_dir,
label_col=args.label_col,
cat_cols=list(args.cat_cols),
n_splits=args.n_splits,
random_seed=args.fold_seed,
)
return {
"args": args,
"clinical": clinical,
"image_dir": args.image_dir,
"clinical_dir": args.clinical_dir,
"label_col": args.label_col,
"cat_cols": list(args.cat_cols),
"image_transform": {
"resize": 256,
"center_crop": 224,
"hflip": True,
"vflip": True,
"rotation": 15,
"color_jitter": (0.1, 0.1, 0.1, 0.05),
},
"image_tower": {
"backbone": args.backbone,
"freeze_ratio": args.freeze_ratio,
"augment": args.img_augment,
"geometry_dim": 0,
"use_se": False, # se_where default is bridge
"se_reduction": args.se_reduction_tower,
"se_pre_norm": args.se_pre_norm_tower,
},
"md_tower": {
"hidden_dim": 128,
"dropout": 0.1,
"use_se": False,
"se_reduction": args.se_reduction_tower,
"se_pre_norm": args.se_pre_norm_tower,
"freeze_ratio": 0.0,
},
"bridge": {
"method": "fusion" if args.fusion_mode == "fused" else "consensus",
"fusion_dim": 256,
"use_se": args.use_se,
"se_reduction": args.se_reduction,
"se_pre_norm": args.se_pre_norm,
},
}
def build_v2_from_config(path: Path) -> Dict[str, Any]:
assembly = assemble_config(path)
imports = resolve_imports(assembly)
if not imports:
raise ValueError("Config did not include any imports.")
clinical = next(iter(imports.values()))
image_loader = _find_loader(assembly, input_type="image")
matrix_loader = _find_loader(assembly, input_type="matrix")
return {
"assembly": assembly,
"clinical": clinical,
"image_loader": image_loader,
"matrix_loader": matrix_loader,
"image_transform_chain": [t.transform_type for t in image_loader.transforms],
}
def _find_loader(assembly, input_type: str):
matches = [
loader for loader in assembly.loaders.values() if loader.input_type == input_type
]
if not matches:
raise ValueError(f"No loader with input_type={input_type!r} found in config.")
if len(matches) > 1:
raise ValueError(f"Multiple loaders with input_type={input_type!r} found.")
return matches[0]
def compare_configs(v1: Dict[str, Any], v2: Dict[str, Any]) -> List[str]:
diffs: List[str] = []
# data sources
v1_rows, v1_cols = v1["clinical"].df.shape
v2_rows, v2_cols = v2["clinical"].df.shape
if v1_rows != v2_rows or v1_cols != v2_cols:
diffs.append(
f"Clinical DF shape mismatch: v1={v1_rows}x{v1_cols}, v2={v2_rows}x{v2_cols}"
)
# loader presence
if not v2.get("image_loader"):
diffs.append("Missing image loader in v2 config.")
if not v2.get("matrix_loader"):
diffs.append("Missing metadata loader in v2 config.")
# transform chain expectations
expected_chain = ["resize", "center_crop", "jitter_bundle"]
if v2.get("image_transform_chain") != expected_chain:
diffs.append(
f"Image transform chain mismatch: v1 expects {expected_chain}, v2 has {v2.get('image_transform_chain')}"
)
# image tower settings
v1_img = v1["image_tower"]
v2_img = _extract_tower(assembly=v2["assembly"], tower_type="image")
_compare_dict(diffs, "ImageTower", v1_img, v2_img)
# metadata tower settings
v1_md = v1["md_tower"]
v2_md = _extract_tower(assembly=v2["assembly"], tower_type="metadata")
_compare_dict(diffs, "MDTower", v1_md, v2_md)
# bridge settings
v2_bridge = _extract_bridge(v2["assembly"])
_compare_dict(diffs, "Bridge", v1["bridge"], v2_bridge)
if not v2["assembly"].classifiers:
diffs.append("Missing classifier node in v2 config.")
# splits
v1_train, v1_val = v1["clinical"].get_split_dfs(0)
sm = PatientFirstSplitManager()
args = SimpleNamespace(
n_splits=v1["args"].n_splits,
fold_seed=v1["args"].fold_seed,
holdout_per_class=v1["args"].holdout_per_class,
holdout_seed=v1["args"].holdout_seed,
eval_mode=v1["args"].eval_mode,
)
splits = sm.build_plans(clinical=v2["clinical"], args=args, profile=None)
v2_train = splits[0].train
v2_val = splits[0].val
if len(v1_train) != len(v2_train) or len(v1_val) != len(v2_val):
diffs.append(
f"Split sizes mismatch: v1 train/val={len(v1_train)}/{len(v1_val)}, "
f"v2 train/val={len(v2_train)}/{len(v2_val)}"
)
return diffs
def _extract_tower(*, assembly, tower_type: str) -> Dict[str, Any]:
towers = [
tower for tower in assembly.towers.values() if tower.tower_type == tower_type
]
if not towers:
raise ValueError(f"No {tower_type} tower found in v2 config.")
if len(towers) > 1:
raise ValueError(f"Multiple {tower_type} towers found in v2 config.")
return towers[0].params
def _extract_bridge(assembly) -> Dict[str, Any]:
if not assembly.bridges:
raise ValueError("No bridge node found in v2 config.")
if len(assembly.bridges) > 1:
raise ValueError("Multiple bridge nodes found in v2 config.")
bridge = next(iter(assembly.bridges.values()))
payload = dict(bridge.params)
payload["method"] = bridge.method
return payload
def _compare_dict(diffs: List[str], label: str, v1: Dict[str, Any], v2: Dict[str, Any]) -> None:
for key, v1_val in v1.items():
v2_val = v2.get(key)
if isinstance(v1_val, tuple):
v1_val = list(v1_val)
if isinstance(v2_val, tuple):
v2_val = list(v2_val)
if v1_val != v2_val:
diffs.append(f"{label} mismatch for {key}: v1={v1_val} v2={v2_val}")
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument(
"--config",
type=Path,
default=Path("hypertower_v2_config.json"),
help="Path to v2 config JSON",
)
parser.add_argument("--samples", type=int, default=8, help="Number of samples to compare")
parser.add_argument("--seed", type=int, default=1234, help="Seed used for deterministic comparisons")
parser.add_argument(
"--image-compare",
choices=["shape", "value"],
default="value",
help="Compare image tensors by shape only or by value",
)
parser.add_argument(
"--no-data-compare",
action="store_true",
help="Skip the data/loader comparison step",
)
parser.add_argument(
"--sample-mode",
choices=["eye", "patient"],
default="eye",
help="Sample mode for V2 loaders (eye-level or patient-level).",
)
parser.add_argument("--train-epochs", type=int, default=2, help="Epochs to run in train comparison.")
parser.add_argument("--train-folds", type=int, default=2, help="Folds to run in train comparison.")
parser.add_argument("--train-batch-size", type=int, default=8, help="Batch size for train comparison.")
parser.add_argument("--max-batches", type=int, default=10, help="Max batches per epoch (train/val).")
parser.add_argument("--loss-tol", type=float, default=0.5, help="Tolerance for loss diffs.")
parser.add_argument("--acc-tol", type=float, default=0.15, help="Tolerance for accuracy diffs.")
parser.add_argument(
"--device",
choices=["auto", "cpu", "cuda"],
default="cpu",
help="Device to use for training comparison.",
)
parser.add_argument(
"--no-train-compare",
action="store_true",
help="Skip the training comparison step.",
)
args = parser.parse_args()
v1 = build_v1_defaults()
v2 = build_v2_from_config(args.config)
diffs = compare_configs(v1, v2)
if diffs:
print("Differences detected:")
for diff in diffs:
print(f"- {diff}")
return 1
print("V1 vs V2 config comparison: OK (settings and loaders match).")
if not args.no_data_compare:
data_diffs = compare_initial_data(
v1,
v2,
samples=args.samples,
seed=args.seed,
compare_mode=args.image_compare,
)
if data_diffs:
print("Differences detected in initial data:")
for diff in data_diffs:
print(f"- {diff}")
return 1
print("Initial data comparison: OK (image/meta/label inputs match).")
if not args.no_train_compare:
train_diffs = compare_training_runs(
v1,
v2,
epochs=args.train_epochs,
folds=args.train_folds,
batch_size=args.train_batch_size,
max_batches=args.max_batches,
seed=args.seed,
loss_tol=args.loss_tol,
acc_tol=args.acc_tol,
sample_mode=args.sample_mode,
device=args.device,
)
if train_diffs:
print("Differences detected in training comparison:")
for diff in train_diffs:
print(f"- {diff}")
return 1
print("Training comparison: OK (metrics within tolerance).")
return 0
def _seed_all(seed: int) -> None:
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
def _build_v1_modules(v1: Dict[str, Any]) -> Dict[str, Any]:
args = v1["args"]
clinical = v1["clinical"]
img_tower = ImageTower(
backbone=args.backbone,
freeze_ratio=args.freeze_ratio,
use_se=False,
se_reduction=args.se_reduction_tower,
se_pre_norm=args.se_pre_norm_tower,
augment=args.img_augment,
geometry_dim=0,
)
md_tower = MDTower(
clinical,
hidden_dim=128,
dropout=0.1,
use_se=False,
se_reduction=args.se_reduction_tower,
se_pre_norm=args.se_pre_norm_tower,
)
bridge = None
if args.fusion_mode == "vote":
bridge = VoteBridge(num_classes=args.num_classes)
else:
bridge = Bridge(
img_dim=img_tower.out_dim,
meta_dim=md_tower.out_dim,
num_classes=args.num_classes,
fusion_dim=256,
mode="fused",
use_se=args.use_se,
se_reduction=args.se_reduction,
se_pre_norm=args.se_pre_norm,
)
return {"image_tower": img_tower, "metadata_tower": md_tower, "bridge": bridge}
def compare_initial_data(
v1: Dict[str, Any],
v2: Dict[str, Any],
*,
samples: int = 8,
seed: int = 1234,
compare_mode: str = "value",
) -> List[str]:
diffs: List[str] = []
v1_modules = _build_v1_modules(v1)
v2_modules = build_model_bundle(v2["assembly"], v2["clinical"])
# Build consistent train split for both datasets
v1_train, _ = v1["clinical"].get_split_dfs(0)
split = PatientSplit(train=v1_train, val=v1_train.iloc[:0], holdout=None)
# V1 dataset
v1_view = _ClinicalView(v1["clinical"], v1_train)
v1_ds = ClinicalDataset(v1_view, v1_modules["image_tower"].transform)
# V2 dataset
if v2_modules.image_transform is None:
diffs.append("V2 image transform could not be built from config.")
return diffs
loader_factory = SlotLoaderFactory(image_transform=v2_modules.image_transform)
v2_loaders = loader_factory.build(
clinical=v2["clinical"],
split=split,
args=SimpleNamespace(batch_size=1),
fold=0,
profile=None,
)
v2_ds = v2_loaders.train.dataset
total = min(samples, len(v1_ds), len(v2_ds))
for idx in range(total):
_seed_all(seed + idx)
v1_item = v1_ds[idx]
_seed_all(seed + idx)
v2_item = v2_ds[idx]
if len(v1_item) == 4:
v1_img, v1_meta, _, v1_label = v1_item
else:
v1_img, v1_meta, v1_label = v1_item
v2_img = v2_item.get("image_1")
v2_meta = v2_item.get("matrix_1")
v2_label = v2_item.get("label_1")
if v2_label is None or int(v2_label) != int(v1_label):
diffs.append(f"Label mismatch at idx {idx}: v1={int(v1_label)} v2={v2_label}")
if v2_meta is None:
diffs.append(f"Missing v2 metadata at idx {idx}")
else:
if not torch.allclose(v1_meta, v2_meta, atol=1e-6, rtol=0.0):
max_diff = float((v1_meta - v2_meta).abs().max().item())
diffs.append(f"Metadata mismatch at idx {idx}: max_abs_diff={max_diff:.6f}")
if v2_img is None:
diffs.append(f"Missing v2 image at idx {idx}")
else:
if tuple(v1_img.shape) != tuple(v2_img.shape):
diffs.append(
f"Image shape mismatch at idx {idx}: v1={tuple(v1_img.shape)} v2={tuple(v2_img.shape)}"
)
elif compare_mode == "value":
max_diff = float((v1_img - v2_img).abs().max().item())
if max_diff > 1e-5:
diffs.append(f"Image tensor mismatch at idx {idx}: max_abs_diff={max_diff:.6f}")
return diffs
def compare_training_runs(
v1: Dict[str, Any],
v2: Dict[str, Any],
*,
epochs: int,
folds: int,
batch_size: int,
max_batches: int,
seed: int,
loss_tol: float,
acc_tol: float,
sample_mode: str,
device: str,
) -> List[str]:
diffs: List[str] = []
if sample_mode != "eye":
diffs.append("Training compare only supports sample_mode='eye' for parity with v1.")
return diffs
if device == "auto":
device = "cuda" if torch.cuda.is_available() else "cpu"
_seed_all(seed)
# Build profile for V2 dataset
profile = build_papila_profile(
patient_col="Patient ID",
label_col=v1["args"].label_col,
sample_mode=sample_mode,
)
# Build splits
sm = PatientFirstSplitManager()
split_args = SimpleNamespace(
n_splits=v1["args"].n_splits,
fold_seed=v1["args"].fold_seed,
holdout_per_class=v1["args"].holdout_per_class,
holdout_seed=v1["args"].holdout_seed,
eval_mode=v1["args"].eval_mode,
)
plans = sm.build_plans(clinical=v2["clinical"], args=split_args, profile=profile)
folds = min(folds, len(plans))
for fold in range(folds):
# Build V1 modules per fold (seeded)
_seed_all(seed + fold * 1000 + 1)
v1_modules = _build_v1_modules(v1)
_move_modules(v1_modules, device)
# Build V2 modules per fold (seeded to match V1 init)
_seed_all(seed + fold * 1000 + 1)
v2_bundle = build_model_bundle(v2["assembly"], v2["clinical"])
if v2_bundle.bridge is None:
diffs.append("V2 model bundle missing bridge.")
return diffs
if isinstance(v2_bundle.bridge, VoteBridge):
diffs.append("V2 bridge is VoteBridge; training compare only supports fusion bridge.")
return diffs
if v2_bundle.image_transform is None:
diffs.append("V2 image transform missing; cannot run training compare.")
return diffs
v2_modules = {
"image_tower": v2_bundle.image_tower,
"metadata_tower": v2_bundle.metadata_tower,
"bridge": v2_bundle.bridge,
}
_move_modules(v2_modules, device)
split = plans[fold]
v1_train = split.train
v1_val = split.val
v1_train_ds = _build_v1_dataset(v1, v1_train, v1_modules["image_tower"].transform)
v1_val_ds = _build_v1_dataset(v1, v1_val, v1_modules["image_tower"].transform)
v2_train_ds = _build_v2_dataset(v2, v1_train, profile, v2_bundle.image_transform)
v2_val_ds = _build_v2_dataset(v2, v1_val, profile, v2_bundle.image_transform)
# Optimizers
v1_opt = torch.optim.Adam(
list(v1_modules["image_tower"].parameters())
+ list(v1_modules["metadata_tower"].parameters())
+ list(v1_modules["bridge"].parameters()),
lr=float(v1["args"].lr),
)
v2_opt = torch.optim.Adam(
list(v2_modules["image_tower"].parameters())
+ list(v2_modules["metadata_tower"].parameters())
+ list(v2_modules["bridge"].parameters()),
lr=float(v1["args"].lr),
)
criterion = nn.CrossEntropyLoss()
for epoch in range(epochs):
_seed_all(seed + fold * 100 + epoch)
v1_train_metrics = _run_epoch_v1(
v1_modules,
v1_train_ds,
v1_opt,
criterion,
device,
batch_size=batch_size,
max_batches=max_batches,
train=True,
seed=seed + fold * 100 + epoch,
)
v2_train_metrics = _run_epoch_v2(
v2_modules,
v2_train_ds,
v2_opt,
criterion,
device,
batch_size=batch_size,
max_batches=max_batches,
train=True,
seed=seed + fold * 100 + epoch,
)
v1_val_metrics = _run_epoch_v1(
v1_modules,
v1_val_ds,
None,
criterion,
device,
batch_size=batch_size,
max_batches=max_batches,
train=False,
seed=seed + fold * 100 + epoch + 777,
)
v2_val_metrics = _run_epoch_v2(
v2_modules,
v2_val_ds,
None,
criterion,
device,
batch_size=batch_size,
max_batches=max_batches,
train=False,
seed=seed + fold * 100 + epoch + 777,
)
print(
f"[fold {fold} epoch {epoch}] "
f"v1 train loss={v1_train_metrics['loss']:.4f} acc={v1_train_metrics['acc']:.4f} | "
f"v2 train loss={v2_train_metrics['loss']:.4f} acc={v2_train_metrics['acc']:.4f}"
)
print(
f"[fold {fold} epoch {epoch}] "
f"v1 val loss={v1_val_metrics['loss']:.4f} acc={v1_val_metrics['acc']:.4f} | "
f"v2 val loss={v2_val_metrics['loss']:.4f} acc={v2_val_metrics['acc']:.4f}"
)
diffs.extend(
_compare_epoch_metrics(
fold,
epoch,
v1_train_metrics,
v2_train_metrics,
v1_val_metrics,
v2_val_metrics,
loss_tol,
acc_tol,
)
)
return diffs
def _compare_epoch_metrics(
fold: int,
epoch: int,
v1_train: Dict[str, float],
v2_train: Dict[str, float],
v1_val: Dict[str, float],
v2_val: Dict[str, float],
loss_tol: float,
acc_tol: float,
) -> List[str]:
diffs: List[str] = []
for split_name, a, b in (
("train", v1_train, v2_train),
("val", v1_val, v2_val),
):
loss_diff = abs(a["loss"] - b["loss"])
acc_diff = abs(a["acc"] - b["acc"])
if loss_diff > loss_tol:
diffs.append(
f"Fold {fold} epoch {epoch} {split_name} loss diff {loss_diff:.4f} (v1={a['loss']:.4f} v2={b['loss']:.4f})"
)
if acc_diff > acc_tol:
diffs.append(
f"Fold {fold} epoch {epoch} {split_name} acc diff {acc_diff:.4f} (v1={a['acc']:.4f} v2={b['acc']:.4f})"
)
return diffs
def _move_modules(modules: Dict[str, Any], device: str) -> None:
for module in modules.values():
if module is not None and hasattr(module, "to"):
module.to(device)
def _build_v1_dataset(v1: Dict[str, Any], df, image_transform) -> ClinicalDataset:
view = _ClinicalView(v1["clinical"], df)
return ClinicalDataset(view, image_transform)
def _build_v2_dataset(v2: Dict[str, Any], df, profile, image_transform) -> SlotDataset:
samples = profile.build_samples(df=df, clinical=v2["clinical"])
return SlotDataset(
samples,
profile.slot_descriptors(),
image_transform=image_transform,
)
def _make_loader(
dataset,
*,
batch_size: int,
shuffle: bool,
seed: int,
collate_fn=None,
) -> torch.utils.data.DataLoader:
g = torch.Generator()
g.manual_seed(seed)
return torch.utils.data.DataLoader(
dataset,
batch_size=batch_size,
shuffle=shuffle,
generator=g,
collate_fn=collate_fn,
)
def _run_epoch_v1(
modules: Dict[str, Any],
dataset: ClinicalDataset,
optimizer: Optional[torch.optim.Optimizer],
criterion: nn.Module,
device: str,
*,
batch_size: int,
max_batches: int,
train: bool,
seed: int,
) -> Dict[str, float]:
_seed_all(seed)
loader = _make_loader(dataset, batch_size=batch_size, shuffle=train, seed=seed)
image_tower = modules["image_tower"]
md_tower = modules["metadata_tower"]
bridge = modules["bridge"]
image_tower.train(train)
md_tower.train(train)
bridge.train(train)
total_loss = 0.0
total_correct = 0
total_count = 0
context = torch.enable_grad() if train else torch.no_grad()
with context:
for step, batch in enumerate(loader):
if step >= max_batches:
break
if len(batch) == 4:
imgs, metas, _, labels = batch
else:
imgs, metas, labels = batch
imgs = imgs.to(device)
metas = metas.to(device)
labels = labels.to(device)
if optimizer is not None:
optimizer.zero_grad()
img_feats = image_tower(imgs)
md_feats = md_tower(metas)
out_fused, _, _ = bridge(img_feats, md_feats)
loss = criterion(out_fused, labels)
if optimizer is not None:
loss.backward()
optimizer.step()
total_loss += float(loss.detach().item()) * labels.size(0)
total_correct += (out_fused.argmax(dim=1) == labels).sum().item()
total_count += labels.size(0)
if total_count == 0:
return {"loss": float("nan"), "acc": float("nan")}
return {"loss": total_loss / total_count, "acc": total_correct / total_count}
def _run_epoch_v2(
modules: Dict[str, Any],
dataset: SlotDataset,
optimizer: Optional[torch.optim.Optimizer],
criterion: nn.Module,
device: str,
*,
batch_size: int,
max_batches: int,
train: bool,
seed: int,
) -> Dict[str, float]:
_seed_all(seed)
loader = _make_loader(
dataset,
batch_size=batch_size,
shuffle=train,
seed=seed,
collate_fn=slot_collate,
)
image_tower = modules["image_tower"]
md_tower = modules["metadata_tower"]
bridge = modules["bridge"]
image_tower.train(train)
md_tower.train(train)
bridge.train(train)
total_loss = 0.0
total_correct = 0
total_count = 0
context = torch.enable_grad() if train else torch.no_grad()
with context:
for step, batch in enumerate(loader):
if step >= max_batches:
break
imgs = batch.get("image_1")
metas = batch.get("matrix_1")
labels = batch.get("label_1")
if imgs is None or metas is None or labels is None:
continue
if not torch.is_tensor(imgs) or not torch.is_tensor(metas):
continue
imgs = imgs.to(device)
metas = metas.to(device)
labels = torch.as_tensor(labels, device=device)
if optimizer is not None:
optimizer.zero_grad()
img_feats = image_tower(imgs)
md_feats = md_tower(metas)
out_fused, _, _ = bridge(img_feats, md_feats)
loss = criterion(out_fused, labels)
if optimizer is not None:
loss.backward()
optimizer.step()
total_loss += float(loss.detach().item()) * labels.size(0)
total_correct += (out_fused.argmax(dim=1) == labels).sum().item()
total_count += labels.size(0)
if total_count == 0:
return {"loss": float("nan"), "acc": float("nan")}
return {"loss": total_loss / total_count, "acc": total_correct / total_count}
if __name__ == "__main__":
raise SystemExit(main())
scripts/testing/test_v2_papila_loaders.py
+135
View File
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Describe PAPILA splits and V2 slot-based loaders."""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
from types import SimpleNamespace
import torch
# Ensure repo root is importable when running as: python3 scripts/test_v2_papila_loaders.py
REPO_ROOT = Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes import build_papila_clinical
from classes.v2 import build_papila_profile, PatientFirstSplitManager, SlotLoaderFactory
def _describe_split(name: str, df, label_col: str) -> str:
if df is None or df.empty:
return f"{name}: empty"
patient_ids = set(df["Patient ID"].tolist())
class_counts = dict(df.groupby(label_col).size().to_dict())
return (
f"{name}: patients={len(patient_ids)} rows={len(df)} "
f"class_rows={class_counts}"
)
def _describe_batch(batch: dict) -> list[str]:
lines = []
for key, val in batch.items():
if isinstance(val, torch.Tensor):
lines.append(f"{key}: tensor shape={tuple(val.shape)} dtype={val.dtype}")
elif isinstance(val, list):
non_none = next((v for v in val if v is not None), None)
lines.append(
f"{key}: list len={len(val)} sample_type={type(non_none).__name__ if non_none is not None else 'None'}"
)
else:
lines.append(f"{key}: {type(val).__name__}")
return lines
def main() -> None:
ap = argparse.ArgumentParser(description="Describe PAPILA splits + V2 slot-based loaders.")
ap.add_argument("--image-dir", default="Papila/FundusImages")
ap.add_argument("--clinical-dir", default="Papila/ClinicalData")
ap.add_argument("--label-col", default="Diagnosis")
ap.add_argument(
"--cat-cols",
nargs="*",
default=["Gender", "Phakic/Pseudophakic"],
help="Categorical columns for PAPILA builder.",
)
ap.add_argument("--n-splits", type=int, default=5)
ap.add_argument("--fold-seed", type=int, default=42)
ap.add_argument("--holdout-per-class", type=int, default=1)
ap.add_argument("--holdout-seed", type=int, default=123)
ap.add_argument("--batch-size", type=int, default=4)
ap.add_argument("--num-workers", type=int, default=0)
ap.add_argument("--fold", type=int, default=0, help="Which fold to inspect in detail.")
ap.add_argument(
"--sample-mode",
choices=["patient", "eye"],
default="patient",
help="Build samples per patient (multi-slot) or per eye (row-level).",
)
args = ap.parse_args()
clinical = build_papila_clinical(
image_dir=args.image_dir,
clinical_dir=args.clinical_dir,
label_col=args.label_col,
cat_cols=args.cat_cols,
n_splits=args.n_splits,
random_seed=args.fold_seed,
)
profile = build_papila_profile(
patient_col="Patient ID",
label_col=args.label_col,
sample_mode=args.sample_mode,
)
print("=== PAPILA profile slots ===")
for key, desc in profile.slot_descriptors().items():
print(f"{key}: kind={desc.kind} required={desc.required} desc={desc.description}")
print("aliases:", profile.semantic_aliases())
split_args = SimpleNamespace(
eval_mode="multiclass",
holdout_per_class=args.holdout_per_class,
holdout_seed=args.holdout_seed,
n_splits=args.n_splits,
fold_seed=args.fold_seed,
)
split_manager = PatientFirstSplitManager(patient_col="Patient ID", label_col=args.label_col)
plans = split_manager.build_plans(clinical=clinical, args=split_args, profile=profile)
print("\n=== Split summaries ===")
for i, split in enumerate(plans):
print(f"fold {i}:")
print(" " + _describe_split("train", split.train, args.label_col))
print(" " + _describe_split("val", split.val, args.label_col))
print(" " + _describe_split("holdout", split.holdout, args.label_col))
if args.fold < 0 or args.fold >= len(plans):
raise SystemExit(f"Requested fold {args.fold} but only {len(plans)} folds are available")
split = plans[args.fold]
loader_factory = SlotLoaderFactory(num_workers=args.num_workers)
loaders = loader_factory.build(
clinical=clinical,
split=split,
args=SimpleNamespace(batch_size=args.batch_size),
fold=args.fold,
profile=profile,
)
print(f"\n=== Loader inspection (fold {args.fold}) ===")
for name, loader in (("train", loaders.train), ("val", loaders.val), ("holdout", loaders.holdout)):
if loader is None:
print(f"{name}: None")
continue
print(f"{name}: batches={len(loader)} batch_size={loader.batch_size}")
batch = next(iter(loader))
for line in _describe_batch(batch):
print(f" {line}")
if __name__ == "__main__":
main()
scripts/testing/test_v2_split_manager.py
+218
View File
@@ -0,0 +1,218 @@
#!/usr/bin/env python3
"""Tiny smoke test for classes.v2.split_manager."""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
from types import SimpleNamespace
import numpy as np
import pandas as pd
# Ensure repo root is importable when running as: python3 scripts/test_v2_split_manager.py
REPO_ROOT = Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from classes import build_papila_clinical
from classes.v2 import build_papila_profile
from classes.v2.split_manager import PatientFirstSplitManager, build_patient_split_plans
class _ClinicalStub:
def __init__(self, df: pd.DataFrame, label_col: str = "Diagnosis") -> None:
self.df = df
self.label_col = label_col
def _make_fake_df(n_patients: int, n_classes: int, label_col: str) -> pd.DataFrame:
rows = []
for pid in range(1, n_patients + 1):
label = (pid - 1) % n_classes
for eye in ("OD", "OS"):
rows.append(
{
"Patient ID": pid,
"eyeID": eye,
label_col: label,
"dummy_feature": float(pid),
}
)
return pd.DataFrame(rows)
def _summarize_fold(
fold: int,
split,
label_col: str,
expected_rows_per_patient: int | None = None,
) -> str:
train_ids = set(split.train["Patient ID"].tolist())
val_ids = set(split.val["Patient ID"].tolist())
holdout_ids = set(split.holdout["Patient ID"].tolist()) if split.holdout is not None else set()
if train_ids & val_ids:
raise RuntimeError(f"Fold {fold}: train/val overlap detected")
if train_ids & holdout_ids:
raise RuntimeError(f"Fold {fold}: train/holdout overlap detected")
if val_ids & holdout_ids:
raise RuntimeError(f"Fold {fold}: val/holdout overlap detected")
if expected_rows_per_patient is not None:
# Used only for synthetic data where we know OD+OS are both present.
for name, df in (("train", split.train), ("val", split.val), ("holdout", split.holdout)):
if df is None or df.empty:
continue
counts = df.groupby("Patient ID").size().unique().tolist()
if counts != [expected_rows_per_patient]:
raise RuntimeError(f"Fold {fold}: {name} has broken per-patient row grouping: {counts}")
train_cls = dict(split.train.groupby(label_col).size().to_dict())
val_cls = dict(split.val.groupby(label_col).size().to_dict())
hold_cls = dict(split.holdout.groupby(label_col).size().to_dict()) if split.holdout is not None else {}
return (
f"fold={fold} "
f"train_patients={len(train_ids)} val_patients={len(val_ids)} holdout_patients={len(holdout_ids)} "
f"train_rows={len(split.train)} val_rows={len(split.val)} holdout_rows={0 if split.holdout is None else len(split.holdout)} "
f"train_class_rows={train_cls} val_class_rows={val_cls} holdout_class_rows={hold_cls}"
)
def _confirm_holdout_consistency_and_exclusion(splits) -> None:
holdout_sets: list[set] = []
val_union: set = set()
for split in splits:
holdout_ids = set(split.holdout["Patient ID"].tolist()) if split.holdout is not None else set()
holdout_sets.append(holdout_ids)
val_union.update(split.val["Patient ID"].tolist())
# A) Holdout should be the same patients across all folds.
baseline = holdout_sets[0] if holdout_sets else set()
for i, holdout_ids in enumerate(holdout_sets):
if holdout_ids != baseline:
raise RuntimeError(
f"Holdout mismatch: fold 0 has {sorted(baseline)}, fold {i} has {sorted(holdout_ids)}"
)
# B) Holdout patients should never appear in any validation/test fold.
overlap = baseline & val_union
if overlap:
raise RuntimeError(f"Holdout patients found in val/test sets: {sorted(overlap)}")
print(
"Holdout checks: OK "
f"(constant across folds, holdout_patients={len(baseline)}, overlap_with_any_val=0)"
)
def main() -> None:
ap = argparse.ArgumentParser(description="Smoke test PatientFirstSplitManager with synthetic data.")
ap.add_argument("--dataset", choices=["papila", "synthetic"], default="papila")
ap.add_argument(
"--patients",
type=int,
default=30,
help="Synthetic mode only: number of fake patients to generate.",
)
ap.add_argument(
"--synthetic-classes",
type=int,
default=3,
help="Synthetic mode only: number of classes to generate.",
)
ap.add_argument("--n-splits", type=int, default=5)
ap.add_argument("--holdout-per-class", type=int, default=1)
ap.add_argument("--fold-seed", type=int, default=42)
ap.add_argument("--holdout-seed", type=int, default=123)
ap.add_argument("--image-dir", default="Papila/FundusImages")
ap.add_argument("--clinical-dir", default="Papila/ClinicalData")
ap.add_argument("--label-col", default="Diagnosis")
ap.add_argument(
"--sample-mode",
choices=["patient", "eye"],
default="patient",
help="Build samples per patient (multi-slot) or per eye (row-level).",
)
ap.add_argument(
"--cat-cols",
nargs="*",
default=["Gender", "Phakic/Pseudophakic"],
help="Categorical columns for PAPILA builder.",
)
args = ap.parse_args()
if args.synthetic_classes < 2:
raise SystemExit("--synthetic-classes must be >= 2")
expected_rows_per_patient: int | None = None
if args.dataset == "papila":
clinical = build_papila_clinical(
image_dir=args.image_dir,
clinical_dir=args.clinical_dir,
label_col=args.label_col,
cat_cols=args.cat_cols,
n_splits=args.n_splits,
random_seed=args.fold_seed,
)
df = clinical.df.copy()
print(
f"Loaded PAPILA dataframe: rows={len(df)} patients={df['Patient ID'].nunique()} "
f"labels={dict(df.groupby(args.label_col).size().to_dict())}"
)
else:
df = _make_fake_df(args.patients, args.synthetic_classes, args.label_col)
clinical = _ClinicalStub(df=df, label_col=args.label_col)
expected_rows_per_patient = 2
print(
f"Loaded synthetic dataframe: rows={len(df)} patients={df['Patient ID'].nunique()} "
f"labels={dict(df.groupby(args.label_col).size().to_dict())}"
)
n_classes = int(df[args.label_col].nunique())
print(f"Detected classes from dataframe: n_classes={n_classes}")
split_args = SimpleNamespace(
eval_mode="multiclass",
holdout_per_class=args.holdout_per_class,
holdout_seed=args.holdout_seed,
n_splits=args.n_splits,
fold_seed=args.fold_seed,
)
manager = PatientFirstSplitManager(patient_col="Patient ID", label_col=args.label_col)
profile = build_papila_profile(
patient_col="Patient ID",
label_col=args.label_col,
sample_mode=args.sample_mode,
)
splits = manager.build_plans(clinical=clinical, args=split_args, profile=profile)
print("=== Adapter split manager output ===")
for fold, split in enumerate(splits):
print(
_summarize_fold(
fold,
split,
label_col=args.label_col,
expected_rows_per_patient=expected_rows_per_patient,
)
)
_confirm_holdout_consistency_and_exclusion(splits)
# Also smoke-test the pure vector API directly.
patient_labels = df.groupby("Patient ID")[args.label_col].first()
plans = build_patient_split_plans(
patient_ids=patient_labels.index.to_numpy(),
patient_labels=patient_labels.to_numpy(),
n_splits=args.n_splits,
seed=args.fold_seed,
holdout_per_class=args.holdout_per_class,
holdout_seed=args.holdout_seed,
)
print(f"\nVector API produced {len(plans)} fold plans.")
print("OK: split manager smoke test passed.")
if __name__ == "__main__":
main()
scripts/utility/check_gpu.py
Executable
+94
View File
@@ -0,0 +1,94 @@
#!/usr/bin/env python3
"""
Quick GPU sanity for PyTorch (CUDA or ROCm).
Prints device visibility, names, memory, and runs a tiny matmul on GPU:0 if available.
Run:
python3 scripts/check_gpu.py
"""
from __future__ import annotations
import os, time, shutil, platform
def _fmt_gb(b: int) -> str:
try:
return f"{b / (1024**3):.2f} GB"
except Exception:
return str(b)
def main():
try:
import torch
except Exception as e:
print(f"torch import failed: {e}")
return
print(f"torch: {getattr(torch, '__version__', 'unknown')}")
print(f"python: {platform.python_version()} on {platform.platform()}")
print(f"CUDA_VISIBLE_DEVICES={os.environ.get('CUDA_VISIBLE_DEVICES', '')!r}")
print(f"HIP_VISIBLE_DEVICES={os.environ.get('HIP_VISIBLE_DEVICES', '')!r}")
print(f"torch.version.cuda={getattr(torch.version, 'cuda', None)}")
print(f"torch.version.hip={getattr(torch.version, 'hip', None)}")
# Apple MPS check (macOS)
if hasattr(torch.backends, 'mps'):
print(f"mps available={torch.backends.mps.is_available()} built={torch.backends.mps.is_built()}")
# CUDA/ROCm check
use_cuda = torch.cuda.is_available()
print(f"cuda available={use_cuda}")
if not use_cuda:
print("No CUDA/ROCm device visible to PyTorch.")
nvsmi = shutil.which('nvidia-smi')
rocmsmi = shutil.which('rocm-smi') or shutil.which('rocminfo')
if nvsmi:
print("nvidia-smi found; ensure your env uses a CUDA-enabled PyTorch build.")
if rocmsmi:
print("ROCm tools found; ensure your env uses a ROCm-enabled PyTorch build.")
print("Tip: activate your conda env and reinstall the GPU build if needed.")
return
# List devices
try:
n = torch.cuda.device_count()
except Exception as e:
print(f"device_count error: {e}")
n = 0
print(f"device_count={n}")
for i in range(n):
try:
name = torch.cuda.get_device_name(i)
except Exception:
name = "?"
try:
props = torch.cuda.get_device_properties(i)
mem = _fmt_gb(getattr(props, 'total_memory', 0))
except Exception:
mem = "?"
print(f" cuda:{i} → {name} | total_memory={mem}")
# Quick matmul on cuda:0
import torch
try:
dev = torch.device('cuda:0')
torch.cuda.synchronize()
a = torch.randn(2048, 2048, device=dev)
b = torch.randn(2048, 2048, device=dev)
t0 = time.time()
c = a @ b
torch.cuda.synchronize()
dt = time.time() - t0
print(f"matmul(2048x2048) on cuda:0 ok in {dt:.3f}s; c.mean={float(c.mean()):.5f}")
del a, b, c
torch.cuda.empty_cache()
except Exception as e:
print(f"matmul on cuda:0 failed: {e}")
# cuDNN info (if CUDA build)
try:
print(f"cudnn available={torch.backends.cudnn.is_available()} version={torch.backends.cudnn.version()}")
except Exception:
pass
if __name__ == "__main__":
main()
scripts/utility/cleanup_grid_runs.sh
Executable
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env bash
# Remove grid-search artifacts (analysis_data/grid_search and models/grid_search).
# Default is a dry run; pass --yes to actually delete.
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
ANALYSIS_DIR="$ROOT_DIR/analysis_data/grid_search"
MODELS_DIR="$ROOT_DIR/models/grid_search"
DRY_RUN=1
for arg in "$@"; do
if [[ "$arg" == "--yes" ]]; then
DRY_RUN=0
fi
done
echo "[cleanup] Target analysis dir: $ANALYSIS_DIR"
echo "[cleanup] Target models dir: $MODELS_DIR"
if [[ $DRY_RUN -eq 1 ]]; then
echo "[cleanup] Dry run only. Nothing deleted. Pass --yes to remove."
exit 0
fi
if [[ -d "$ANALYSIS_DIR" ]]; then
echo "[cleanup] Removing $ANALYSIS_DIR"
rm -rf "$ANALYSIS_DIR"
else
echo "[cleanup] Analysis dir not found; skipping."
fi
if [[ -d "$MODELS_DIR" ]]; then
echo "[cleanup] Removing $MODELS_DIR"
rm -rf "$MODELS_DIR"
else
echo "[cleanup] Models dir not found; skipping."
fi
echo "[cleanup] Done."