rpotter6298
af813bbb62
- Updated GTGeometryLoader to streamline geometry vector computation and caching. - Introduced UNetGeometryLoader for UNet-derived geometry vectors. - Added sample collection method in PapilaBundle for better data handling. - Refined GeometrySegEncoder and ImageEncoder to utilize new sample collection. - Enhanced distributed client with heartbeat mechanism for improved job tracking. - Added new configuration files for UNet-derived geometry integration.
710 lines
27 KiB
Python
710 lines
27 KiB
Python
"""v4papila — self-contained PAPILA data module.
|
|
|
|
Public contract (v4 orchestrator interface)
|
|
-------------------------------------------
|
|
bundle = build_data(args: dict) -> PapilaBundle
|
|
|
|
PapilaBundle exposes:
|
|
.df full preprocessed DataFrame (for split building)
|
|
.label_col, .patient_col, .feature_dim
|
|
.id_names tuple of semantic names for each entity_id slot
|
|
e.g. ("patient_id", "eye") — used by orchestrator for logging
|
|
.matrix ClinicalDataView (all eyes)
|
|
.matrix.od / .matrix.os scoped views (OD or OS only)
|
|
.image ImageDataView (all eyes)
|
|
.image.od / .image.os scoped views
|
|
.build_shells(df, *, level) -> LoaderShell
|
|
|
|
DataView interface (consumed by towers' get_sample)
|
|
----------------------------------------------------
|
|
Both views accept positional id slots (*ids) matching entity_id tuple positions.
|
|
Semantic names for each position are in view.id_names.
|
|
|
|
ClinicalDataView:
|
|
.feature_dim
|
|
.id_names e.g. ("patient_id", "eye")
|
|
.vectorize_entity(*ids) -> np.ndarray
|
|
.side_map -> dict mapping generic keys {"a", "b"} to id_1 values
|
|
.od, .os -> scoped ClinicalDataView
|
|
|
|
ImageDataView:
|
|
.id_names e.g. ("patient_id", "eye")
|
|
.get_image_path(*ids) -> Path
|
|
.load_image(*ids) -> PIL.Image
|
|
.side_map -> dict mapping generic keys {"a", "b"} to id_1 values
|
|
.od, .os -> scoped ImageDataView
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from functools import cached_property
|
|
from pathlib import Path
|
|
from typing import Callable, Dict, List, Optional
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
from PIL import Image
|
|
|
|
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
if str(_REPO_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(_REPO_ROOT))
|
|
|
|
from v4.classes.loaders.image_loader import CachedImageLoader, call_preprocessor
|
|
from v4.classes.dataset import DataBundle, LoaderShell, ShellEntry
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pachymetry → IOP correction (PAPILA Table 3)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_PACHY_TABLE: Dict[int, int] = {
|
|
475: +5, 485: +4, 495: +4, 505: +3, 515: +2,
|
|
525: +1, 535: +1, 545: 0, 555: -1, 565: -1,
|
|
575: -2, 585: -3, 595: -4, 605: -4, 615: -5,
|
|
}
|
|
_PACHY_KEYS = np.array(sorted(_PACHY_TABLE.keys()))
|
|
|
|
|
|
def _nearest_pachy_key(x: float) -> int:
|
|
return int(_PACHY_KEYS[int(np.argmin(np.abs(_PACHY_KEYS - float(x))))])
|
|
|
|
|
|
def _fit_perkins_converter(
|
|
frames: List[pd.DataFrame], method: str
|
|
) -> Callable[[float, Optional[float]], float]:
|
|
combined = pd.concat(frames, ignore_index=True)
|
|
paired = combined.dropna(subset=["Pneumatic", "Perkins"])
|
|
if len(paired) == 0:
|
|
raise ValueError("No paired Pneumatic+Perkins rows; cannot fit converter.")
|
|
pneumatic = paired["Pneumatic"].values.astype(float)
|
|
perkins = paired["Perkins"].values.astype(float)
|
|
|
|
if method == "ratio":
|
|
ratio = float((pneumatic / perkins).mean())
|
|
def _conv(p: float, pachy: Optional[float] = None) -> float:
|
|
return p * ratio
|
|
return _conv
|
|
|
|
elif method == "ols":
|
|
from scipy import stats as _stats
|
|
slope, intercept, *_ = _stats.linregress(perkins, pneumatic)
|
|
slope, intercept = float(slope), float(intercept)
|
|
def _conv(p: float, pachy: Optional[float] = None) -> float:
|
|
return p * slope + intercept
|
|
return _conv
|
|
|
|
elif method == "lad":
|
|
from scipy import stats as _stats
|
|
from scipy.optimize import minimize as _minimize
|
|
slope0, intercept0, *_ = _stats.linregress(perkins, pneumatic)
|
|
def _lad_loss(params):
|
|
a, b = params
|
|
return np.abs(pneumatic - (a * perkins + b)).mean()
|
|
res = _minimize(_lad_loss, x0=[slope0, intercept0], method="Nelder-Mead")
|
|
slope, intercept = float(res.x[0]), float(res.x[1])
|
|
def _conv(p: float, pachy: Optional[float] = None) -> float:
|
|
return p * slope + intercept
|
|
return _conv
|
|
|
|
elif method == "multi":
|
|
from numpy.linalg import lstsq as _lstsq
|
|
pm = combined.dropna(subset=["Pneumatic", "Perkins", "Pachymetry"])
|
|
if len(pm) == 0:
|
|
raise ValueError("No Pneumatic+Perkins+Pachymetry rows; cannot fit multi.")
|
|
pneu = pm["Pneumatic"].values.astype(float)
|
|
perk = pm["Perkins"].values.astype(float)
|
|
pv = pm["Pachymetry"].values.astype(float)
|
|
X = np.column_stack([perk, pv, np.ones(len(perk))])
|
|
coeffs, *_ = _lstsq(X, pneu, rcond=None)
|
|
slope, pachy_coef, intercept = float(coeffs[0]), float(coeffs[1]), float(coeffs[2])
|
|
fallback = float(pv.mean())
|
|
def _conv(p: float, pachy: Optional[float] = None) -> float:
|
|
pval = pachy if (pachy is not None and not np.isnan(pachy)) else fallback
|
|
return p * slope + pachy_coef * pval + intercept
|
|
return _conv
|
|
|
|
else:
|
|
raise ValueError(f"Unknown iop_corr_method: {method!r}. Choose ratio/ols/lad/multi.")
|
|
|
|
|
|
def _pick_iop(row: pd.Series, converter: Callable) -> float:
|
|
pneumatic = row.get("Pneumatic", np.nan)
|
|
if not pd.isna(pneumatic):
|
|
return float(pneumatic)
|
|
perkins = row.get("Perkins", np.nan)
|
|
if pd.isna(perkins):
|
|
return np.nan
|
|
pachy = row.get("Pachymetry", np.nan)
|
|
return converter(float(perkins), None if pd.isna(pachy) else float(pachy))
|
|
|
|
|
|
def _correct_iop(raw_iop: float, pachy: float) -> float:
|
|
if pd.isna(raw_iop):
|
|
return np.nan
|
|
if pd.isna(pachy):
|
|
return float(raw_iop)
|
|
key = _nearest_pachy_key(float(pachy))
|
|
return float(raw_iop) + float(_PACHY_TABLE[key])
|
|
|
|
|
|
def _apply_iop_and_drop_md(
|
|
df: pd.DataFrame, converter: Callable, drop_raw: bool = False
|
|
) -> pd.DataFrame:
|
|
df["IOP_raw"] = df.apply(lambda row: _pick_iop(row, converter), axis=1)
|
|
pachy = df.get("Pachymetry", pd.Series(np.nan, index=df.index))
|
|
df["IOP_corr"] = [
|
|
_correct_iop(r, p) for r, p in zip(df["IOP_raw"].values, pachy.values)
|
|
]
|
|
drop = [c for c in ("Pneumatic", "Perkins", "VF_MD") if c in df.columns]
|
|
if drop_raw:
|
|
drop.append("IOP_raw")
|
|
if drop:
|
|
df.drop(columns=drop, inplace=True)
|
|
return df
|
|
|
|
|
|
def _canonicalize_eye_column(df: pd.DataFrame) -> None:
|
|
if "eyeID" in df.columns:
|
|
src = "eyeID"
|
|
else:
|
|
src = next((c for c in df.columns if "eye" in c.lower()), None)
|
|
if src is None:
|
|
df["eyeID"] = "OS"
|
|
return
|
|
|
|
def norm(v):
|
|
if pd.isna(v):
|
|
return None
|
|
x = str(v).strip().upper()
|
|
if x in {"OS", "L", "LEFT", "0"}: return "OS"
|
|
if x in {"OD", "R", "RIGHT", "1"}: return "OD"
|
|
try:
|
|
num = int(float(x))
|
|
return "OD" if num % 2 == 1 else "OS"
|
|
except Exception:
|
|
return None
|
|
|
|
mapped = df[src].map(norm)
|
|
uniq = {u for u in mapped.dropna().unique().tolist()}
|
|
if not uniq.issubset({"OS", "OD"}):
|
|
raise ValueError(f"eyeID must be binary; found {sorted(uniq)}")
|
|
df["eyeID"] = mapped.fillna("OS")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# DataView classes
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class ClinicalDataView:
|
|
"""Tabular feature view over a (possibly eye-filtered) clinical DataFrame.
|
|
|
|
Exposes feature_dim and vectorize_entity so towers can retrieve
|
|
feature vectors by entity identity without knowing about the DataFrame.
|
|
|
|
Scoped views (OD or OS only) are accessed via .od and .os properties.
|
|
|
|
id_names gives semantic labels for each positional slot in an entity_id tuple,
|
|
e.g. ("patient_id", "eye"). The orchestrator uses this for logging without
|
|
needing to know PAPILA-specific field names itself.
|
|
"""
|
|
|
|
# PAPILA canonical side keys used in ShellEntry entity_ids
|
|
SIDE_A = "OD"
|
|
SIDE_B = "OS"
|
|
|
|
# Semantic name for each entity_id position (id_0, id_1, ...)
|
|
id_names: tuple[str, ...] = ("patient_id", "eye")
|
|
|
|
def __init__(
|
|
self,
|
|
df: pd.DataFrame,
|
|
patient_col: str,
|
|
scalar_cols: list[str],
|
|
cat_cols: list[str],
|
|
scalar_stats: dict,
|
|
cat_maps: dict,
|
|
*,
|
|
eye_filter: str | None = None, # "OD", "OS", or None (all eyes)
|
|
):
|
|
self._df = df
|
|
self.patient_col = patient_col
|
|
self.scalar_cols = scalar_cols
|
|
self.cat_cols = cat_cols
|
|
self.scalar_stats = scalar_stats
|
|
self.cat_maps = cat_maps
|
|
self._eye_filter = eye_filter
|
|
|
|
# Build a (patient_id, eyeID) → row index for fast lookup
|
|
if "eyeID" in df.columns:
|
|
self._idx = df.set_index([patient_col, "eyeID"])
|
|
else:
|
|
self._idx = df.set_index(patient_col)
|
|
|
|
# ── Public API ──────────────────────────────────────────────────────────
|
|
|
|
@property
|
|
def feature_dim(self) -> int:
|
|
n_scalar = len(self.scalar_cols)
|
|
n_cat = sum(len(m) for m in self.cat_maps.values())
|
|
return n_scalar + n_cat + n_scalar # scalars + one-hots + missing flags
|
|
|
|
def vectorize_entity(self, *ids) -> np.ndarray:
|
|
"""Return the feature vector for an entity identified by positional ids.
|
|
|
|
Positional slots match entity_id tuple positions (see id_names).
|
|
For PAPILA: ids = (id_0, id_1) = (patient_id, eye).
|
|
"""
|
|
try:
|
|
row = self._idx.loc[ids if len(ids) > 1 else ids[0]].copy()
|
|
except KeyError:
|
|
names = self.id_names[:len(ids)]
|
|
raise KeyError(
|
|
f"ClinicalDataView: no row found for {dict(zip(names, ids))}"
|
|
)
|
|
# set_index removes index-level columns from the row; restore any that
|
|
# _vectorize_row needs (e.g. eyeID is a cat feature AND an index level)
|
|
idx_names = (self._idx.index.names
|
|
if hasattr(self._idx.index, 'names')
|
|
else [self._idx.index.name])
|
|
for name, val in zip(idx_names, ids if len(ids) > 1 else [ids[0]]):
|
|
if name not in row.index:
|
|
row[name] = val
|
|
return self._vectorize_row(row)
|
|
|
|
# ── Scoped views ─────────────────────────────────────────────────────────
|
|
|
|
@property
|
|
def side_map(self) -> dict[str, str]:
|
|
"""Generic side-key → dataset side string. Towers use this for patient-level shells."""
|
|
return {"a": self.SIDE_A, "b": self.SIDE_B}
|
|
|
|
@cached_property
|
|
def od(self) -> "ClinicalDataView":
|
|
return self._scoped(self.SIDE_A)
|
|
|
|
@cached_property
|
|
def os(self) -> "ClinicalDataView":
|
|
return self._scoped(self.SIDE_B)
|
|
|
|
def _scoped(self, eye: str) -> "ClinicalDataView":
|
|
sub = self._df[self._df["eyeID"] == eye].reset_index(drop=True)
|
|
return ClinicalDataView(
|
|
df=sub,
|
|
patient_col=self.patient_col,
|
|
scalar_cols=self.scalar_cols,
|
|
cat_cols=self.cat_cols,
|
|
scalar_stats=self.scalar_stats,
|
|
cat_maps=self.cat_maps,
|
|
eye_filter=eye,
|
|
)
|
|
|
|
# ── Internal ─────────────────────────────────────────────────────────────
|
|
|
|
def _vectorize_row(self, row: pd.Series) -> np.ndarray:
|
|
feats: list[float] = []
|
|
miss: list[float] = []
|
|
for col in self.scalar_cols:
|
|
v = pd.to_numeric(row.get(col), errors="coerce")
|
|
if pd.isna(v):
|
|
miss.append(1.0)
|
|
v = self.scalar_stats[col]["median"]
|
|
else:
|
|
miss.append(0.0)
|
|
lo = self.scalar_stats[col]["min"]
|
|
hi = self.scalar_stats[col]["max"]
|
|
feats.append((float(v) - lo) / (hi - lo) if hi > lo else 0.0)
|
|
for col in self.cat_cols:
|
|
mapping = self.cat_maps[col]
|
|
one = [0.0] * len(mapping)
|
|
key = row.get(col)
|
|
one[mapping.get(key, 0)] = 1.0
|
|
feats.extend(one)
|
|
feats.extend(miss)
|
|
return np.asarray(feats, dtype=np.float32)
|
|
|
|
|
|
class ImageDataView:
|
|
"""Image path and loading view over a (possibly eye-filtered) DataFrame.
|
|
|
|
Provides get_image_path and load_image keyed by positional id slots.
|
|
Scoped views (.od, .os) are available for single-side towers.
|
|
The optional image_cache is a shared CachedImageLoader for the run.
|
|
|
|
id_names gives semantic labels for each positional slot in an entity_id tuple,
|
|
e.g. ("patient_id", "eye").
|
|
"""
|
|
|
|
SIDE_A = "OD"
|
|
SIDE_B = "OS"
|
|
|
|
id_names: tuple[str, ...] = ("patient_id", "eye")
|
|
|
|
def __init__(
|
|
self,
|
|
df: pd.DataFrame,
|
|
patient_col: str,
|
|
image_dir: str,
|
|
filename_template: str,
|
|
preprocessor: Callable | None = None,
|
|
image_cache: CachedImageLoader | None = None,
|
|
*,
|
|
eye_filter: str | None = None,
|
|
):
|
|
self._df = df
|
|
self.patient_col = patient_col
|
|
self.image_dir = Path(image_dir)
|
|
self.filename_template = filename_template
|
|
self.preprocessor = preprocessor
|
|
self.image_cache = image_cache
|
|
self._eye_filter = eye_filter
|
|
|
|
if "eyeID" in df.columns:
|
|
self._idx = df.set_index([patient_col, "eyeID"])
|
|
else:
|
|
self._idx = df.set_index(patient_col)
|
|
|
|
# ── Public API ──────────────────────────────────────────────────────────
|
|
|
|
def get_image_path(self, *ids) -> Path:
|
|
"""Return the image path for an entity identified by positional ids.
|
|
|
|
For PAPILA: ids = (id_0, id_1) = (patient_id, eye).
|
|
"""
|
|
id_0, id_1 = ids # PAPILA always uses two slots
|
|
try:
|
|
row = self._idx.loc[(id_0, id_1)]
|
|
except KeyError:
|
|
names = self.id_names[:len(ids)]
|
|
raise KeyError(
|
|
f"ImageDataView: no row for {dict(zip(names, ids))}"
|
|
)
|
|
pid = int(row[self.patient_col]) if self.patient_col in row.index else int(id_0)
|
|
return self.image_dir / self.filename_template.format(pid=pid, eye=id_1)
|
|
|
|
def load_image(self, *ids) -> Image.Image:
|
|
"""Load image for an entity identified by positional ids."""
|
|
path = self.get_image_path(*ids)
|
|
if self.image_cache is not None:
|
|
return self.image_cache.load(path, preprocessor=self.preprocessor)
|
|
img = Image.open(path).convert("RGB")
|
|
if self.preprocessor is not None:
|
|
img = call_preprocessor(self.preprocessor, img, path)
|
|
return img
|
|
|
|
# ── Scoped views ─────────────────────────────────────────────────────────
|
|
|
|
@cached_property
|
|
def od(self) -> "ImageDataView":
|
|
return self._scoped(self.SIDE_A)
|
|
|
|
@cached_property
|
|
def os(self) -> "ImageDataView":
|
|
return self._scoped(self.SIDE_B)
|
|
|
|
@property
|
|
def side_map(self) -> dict[str, str]:
|
|
"""Generic side-key → dataset side string. Towers use this for patient-level shells."""
|
|
return {"a": self.SIDE_A, "b": self.SIDE_B}
|
|
|
|
def _scoped(self, eye: str) -> "ImageDataView":
|
|
sub = self._df[self._df["eyeID"] == eye].reset_index(drop=True)
|
|
return ImageDataView(
|
|
df=sub,
|
|
patient_col=self.patient_col,
|
|
image_dir=str(self.image_dir),
|
|
filename_template=self.filename_template,
|
|
preprocessor=self.preprocessor,
|
|
image_cache=self.image_cache,
|
|
eye_filter=eye,
|
|
)
|
|
|
|
# ── Geometry hook ─────────────────────────────────────────────────────────
|
|
|
|
def _resolve_paths(self, kwargs: dict) -> dict:
|
|
"""Resolve any *_dir / *_path kwargs against the repo root."""
|
|
repo_root = Path(__file__).resolve().parents[3]
|
|
out = {}
|
|
for k, v in kwargs.items():
|
|
if (k.endswith("_dir") or k.endswith("_path")) and v is not None:
|
|
p = Path(v)
|
|
out[k] = str(repo_root / p) if not p.is_absolute() else v
|
|
else:
|
|
out[k] = v
|
|
return out
|
|
|
|
def build_geometry_loader(self, source: str, **kwargs):
|
|
"""Return a geometry-vector loader (delegates to fundus_images)."""
|
|
from v4.classes.profiles.fundus_images import build_geometry_loader as _build
|
|
return _build(source, **self._resolve_paths(kwargs))
|
|
|
|
def build_seg_map_loader(self, source: str, **kwargs):
|
|
"""Return a seg-map loader (delegates to fundus_images)."""
|
|
from v4.classes.profiles.fundus_images import build_seg_map_loader as _build
|
|
return _build(source, **self._resolve_paths(kwargs))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# PapilaBundle — the v4 DataBundle returned by build_data
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class PapilaBundle:
|
|
"""V4 DataBundle for PAPILA.
|
|
|
|
Wraps the v3 DataBundle for backward compatibility (df, feature_dim,
|
|
vectorize_row, get_image_path, patient_col, label_col) while adding
|
|
the v4 DataView interface and build_shells().
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
bundle: DataBundle,
|
|
image_dir: str,
|
|
preprocessor: Callable | None = None,
|
|
image_cache: CachedImageLoader | None = None,
|
|
):
|
|
self._bundle = bundle
|
|
self._image_dir = image_dir
|
|
|
|
# ── ClinicalDataView (all eyes) ──────────────────────────────────────
|
|
self.matrix = ClinicalDataView(
|
|
df=bundle.df,
|
|
patient_col=bundle.patient_col,
|
|
scalar_cols=bundle.scalar_cols,
|
|
cat_cols=bundle.cat_cols,
|
|
scalar_stats=bundle.scalar_stats,
|
|
cat_maps=bundle.cat_maps,
|
|
)
|
|
|
|
# ── ImageDataView (all eyes) ─────────────────────────────────────────
|
|
self.image = ImageDataView(
|
|
df=bundle.df,
|
|
patient_col=bundle.patient_col,
|
|
image_dir=image_dir,
|
|
filename_template=bundle.filename_template,
|
|
preprocessor=preprocessor,
|
|
image_cache=image_cache,
|
|
)
|
|
|
|
# ── Entity-id metadata (for orchestrator logging) ────────────────────────
|
|
|
|
@property
|
|
def id_names(self) -> tuple[str, ...]:
|
|
"""Semantic names for each entity_id position, e.g. ('patient_id', 'eye').
|
|
|
|
Orchestrators use this to decode entity_ids for logging without
|
|
hardcoding dataset-specific field names.
|
|
"""
|
|
return self.matrix.id_names # both views share the same structure
|
|
|
|
# ── Identity column registry (for orchestrator split_identity_level) ────────
|
|
|
|
@property
|
|
def identity_cols(self) -> list[str]:
|
|
"""Ordered list of grouping columns, one per identity level.
|
|
|
|
identity_level=1 → identity_cols[0] → patient column (group by patient)
|
|
identity_level=2 → identity_cols[1] → eye column (group by patient+eye)
|
|
"""
|
|
return [self._bundle.patient_col, "eyeID"]
|
|
|
|
# ── Sample collection (for tower early_pass) ─────────────────────────────
|
|
|
|
def collect_samples(self, df: pd.DataFrame | None) -> list[tuple]:
|
|
"""Build (pid, eye, image_path) tuples from a split DataFrame.
|
|
|
|
Used by tower early_pass implementations that need per-eye image paths
|
|
(UNet inference, contour rasterisation, etc.). Returns [] for an
|
|
empty/None df.
|
|
"""
|
|
if df is None or len(df) == 0:
|
|
return []
|
|
pc = self._bundle.patient_col
|
|
out: list[tuple] = []
|
|
for _, row in df.iterrows():
|
|
pid = int(row[pc])
|
|
eye = str(row.get("eyeID", "OD"))
|
|
out.append((pid, eye, self.image.get_image_path(pid, eye)))
|
|
return out
|
|
|
|
# ── Backward-compat delegates ────────────────────────────────────────────
|
|
|
|
@property
|
|
def df(self) -> pd.DataFrame:
|
|
return self._bundle.df
|
|
|
|
@property
|
|
def label_col(self) -> str:
|
|
return self._bundle.label_col
|
|
|
|
@property
|
|
def patient_col(self) -> str:
|
|
return self._bundle.patient_col
|
|
|
|
@property
|
|
def feature_dim(self) -> int:
|
|
return self._bundle.feature_dim
|
|
|
|
def vectorize_row(self, row: pd.Series) -> np.ndarray:
|
|
return self._bundle.vectorize_row(row)
|
|
|
|
def get_image_path(self, row: pd.Series):
|
|
return self._bundle.get_image_path(row)
|
|
|
|
# ── Shell building ───────────────────────────────────────────────────────
|
|
|
|
def build_shells(
|
|
self,
|
|
df: pd.DataFrame,
|
|
*,
|
|
level: str = "eye",
|
|
label_filter: list[int] | None = None,
|
|
eye_filter: str | None = None,
|
|
) -> LoaderShell:
|
|
"""Build a LoaderShell from a split DataFrame.
|
|
|
|
level="eye" — one ShellEntry per eye row.
|
|
entity_id = (patient_id, side_key)
|
|
e.g. (42, "OD") or (42, "OS")
|
|
|
|
level="patient" — one ShellEntry per patient.
|
|
entity_id = (patient_id,) — 1-tuple
|
|
Towers that need both sides use data_view.side_map
|
|
to assemble them in get_sample.
|
|
Patients missing either eye are excluded.
|
|
"""
|
|
pc = self.patient_col
|
|
lc = self.label_col
|
|
|
|
if label_filter is not None:
|
|
df = df[df[lc].isin(label_filter)]
|
|
if eye_filter is not None and "eyeID" in df.columns:
|
|
df = df[df["eyeID"] == eye_filter]
|
|
|
|
entries: list[ShellEntry] = []
|
|
|
|
if level == "eye":
|
|
for _, row in df.iterrows():
|
|
pid = int(row[pc])
|
|
label = int(row[lc])
|
|
side = str(row.get("eyeID", "OD"))
|
|
entries.append(ShellEntry(entity_id=(pid, side), label=label))
|
|
|
|
elif level == "patient":
|
|
for pid, grp in df.groupby(pc):
|
|
if "eyeID" in grp.columns:
|
|
eyes = set(grp["eyeID"].unique())
|
|
if "OD" not in eyes or "OS" not in eyes:
|
|
continue
|
|
label_mode = grp[lc].mode()
|
|
label = int(label_mode.iloc[0]) if not label_mode.empty else int(grp[lc].iloc[0])
|
|
entries.append(ShellEntry(entity_id=(int(pid),), label=label))
|
|
|
|
else:
|
|
raise ValueError(f"Unknown shell level: {level!r}. Choose 'eye' or 'patient'.")
|
|
|
|
return LoaderShell(entries=entries)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Resolve helper — used by the orchestrator to inject DataView into towers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def resolve_data_source(bundle: PapilaBundle, path: str):
|
|
"""Resolve a dot-path data source string against a PapilaBundle.
|
|
|
|
Examples
|
|
--------
|
|
"matrix" → bundle.matrix
|
|
"matrix.od" → bundle.matrix.od
|
|
"image" → bundle.image
|
|
"image.os" → bundle.image.os
|
|
"""
|
|
obj = bundle
|
|
for part in path.split("."):
|
|
obj = getattr(obj, part)
|
|
return obj
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Public contract: build_data(args: dict) -> PapilaBundle
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_DEFAULT_CAT_COLS = ["Gender", "Phakic/Pseudophakic"]
|
|
|
|
|
|
def build_data(args: dict) -> PapilaBundle:
|
|
"""Build and return a PapilaBundle.
|
|
|
|
args keys
|
|
---------
|
|
image_dir (required)
|
|
clinical_dir (required)
|
|
label_col (default: "Diagnosis")
|
|
iop_corr_method (default: "ratio")
|
|
iop_drop_raw (default: False)
|
|
exclude_cols (default: [])
|
|
cat_cols (default: ["Gender", "Phakic/Pseudophakic"])
|
|
n_splits (default: 5)
|
|
random_seed (default: 42)
|
|
in_memory_cache (default: False) — enable shared image cache for the run
|
|
"""
|
|
image_dir = args["image_dir"]
|
|
clinical_dir = args["clinical_dir"]
|
|
label_col = args.get("label_col", "Diagnosis")
|
|
iop_method = args.get("iop_corr_method", "ratio")
|
|
iop_drop_raw = bool(args.get("iop_drop_raw", False))
|
|
exclude_cols = list(args.get("exclude_cols", []))
|
|
cat_cols = list(args.get("cat_cols", _DEFAULT_CAT_COLS))
|
|
n_splits = int(args.get("n_splits", 5))
|
|
random_seed = int(args.get("random_seed", 42))
|
|
use_cache = bool(args.get("in_memory_cache", False))
|
|
|
|
effective_cat = [c for c in cat_cols if c not in exclude_cols]
|
|
|
|
bundle = DataBundle(
|
|
image_dir=image_dir,
|
|
clinical_dir=clinical_dir,
|
|
label_col=label_col,
|
|
patient_col="Patient ID",
|
|
cat_cols=effective_cat,
|
|
n_splits=n_splits,
|
|
random_seed=random_seed,
|
|
filename_template="RET{pid:03d}{eye}.jpg",
|
|
)
|
|
|
|
od = pd.read_excel(f"{clinical_dir}/patient_data_od.xlsx", header=1)
|
|
od["eyeID"] = "OD"
|
|
os_ = pd.read_excel(f"{clinical_dir}/patient_data_os.xlsx", header=1)
|
|
os_["eyeID"] = "OS"
|
|
|
|
for frame in (od, os_):
|
|
if "Patient ID" not in frame.columns and "ID" in frame.columns:
|
|
frame.rename(columns={"ID": "Patient ID"}, inplace=True)
|
|
frame["Patient ID"] = (
|
|
frame["Patient ID"].astype(str).str.extract(r"(\d+)")[0].astype(int)
|
|
)
|
|
_canonicalize_eye_column(frame)
|
|
|
|
bundle.add_df(od, id_column="ID", exclude_cols=exclude_cols or None)
|
|
bundle.add_df(os_, id_column="ID", exclude_cols=exclude_cols or None)
|
|
|
|
converter = _fit_perkins_converter(bundle.frames, method=iop_method)
|
|
for i in range(len(bundle.frames)):
|
|
bundle.frames[i] = _apply_iop_and_drop_md(
|
|
bundle.frames[i], converter=converter, drop_raw=iop_drop_raw
|
|
)
|
|
|
|
bundle._refresh_master_df(exclude_cols=exclude_cols or None)
|
|
bundle._infer_or_validate_feature_types(exclude_cols=exclude_cols or None)
|
|
bundle._compute_numeric_stats()
|
|
bundle._build_cat_maps()
|
|
bundle._compute_feature_dim()
|
|
|
|
image_cache = CachedImageLoader() if use_cache else None
|
|
|
|
return PapilaBundle(
|
|
bundle=bundle,
|
|
image_dir=image_dir,
|
|
image_cache=image_cache,
|
|
)
|