added better memory caching, multithreaded processing, cleanup scripts dir

2026-03-13 08:08:36 +01:00
parent 8282461a23
commit 7ea85d5426
39 changed files with 1850 additions and 1998 deletions
@@ -0,0 +1,306 @@
+"""Utilities for preparing REFUGE (REFUGE1/REFUGE2) datasets.
+
+Builds a unified manifest across all provided splits (REFUGE1 train/val/test
+and REFUGE2 validation/test), exposing image paths, glaucoma labels, disc/cup
+masks, and fovea coordinates so downstream segmentation/classification modules
+can operate without additional bookkeeping.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Iterable, List, Optional, Tuple
+
+import pandas as pd
+
+
+@dataclass
+class RefugeSample:
+    """Lightweight container describing a REFUGE sample."""
+
+    sample_id: str
+    dataset: str
+    split: str
+    image_path: Path
+    label: Optional[int]
+    device: Optional[str]
+    mask_path: Optional[Path]
+    fovea_coord: Optional[Tuple[float, float]]
+
+
+class RefugePreprocessing:
+    """Builds manifests and provides shared helpers for REFUGE workflows.
+
+    Responsibilities:
+      * scan the REFUGE directory structure and build a consistent manifest
+        (train/val/test, device vendor, ground-truth labels)
+      * expose convenience loaders for raw RGB frames, OD/OC masks, and
+        optional fovea landmarks
+      * compute geometric metadata (disc centres, diameters) so downstream
+        stages can crop ROIs lazily instead of storing pre-rendered tiles
+    """
+
+    def __init__(self, root_dir: Path | str) -> None:
+        self.root_dir = Path(root_dir)
+        self._manifest = None  # populated by build_manifest()
+
+    # ------------------------------------------------------------------
+    # Manifest handling
+    # ------------------------------------------------------------------
+    def build_manifest(self, refresh: bool = False) -> Iterable[RefugeSample]:
+        """Return an iterable of :class:`RefugeSample` records.
+
+        Parameters
+        ----------
+        refresh:
+            when True, force a rescan of the filesystem instead of reusing the
+            cached manifest.
+
+        Returns
+        -------
+        Iterable[RefugeSample]
+            A sequence containing one entry per sample in the REFUGE datasets.
+
+        Notes
+        -----
+        The actual manifest-building logic will live here: parsing the
+        directory structure, reading any provided CSV/Excel metadata, and
+        aligning masks/labels. For now, this method raises ``NotImplementedError``
+        so callers are reminded to hook it up before use.
+        """
+
+        if self._manifest is not None and not refresh:
+            return self._manifest
+
+        manifest: List[RefugeSample] = []
+
+        manifest.extend(self._collect_refuge1_train())
+        manifest.extend(self._collect_refuge1_val())
+        manifest.extend(self._collect_refuge1_test())
+        manifest.extend(self._collect_refuge2_val())
+        manifest.extend(self._collect_refuge2_test())
+
+        self._manifest = manifest
+        return self._manifest
+
+    # ------------------------------------------------------------------
+    # Accessors for downstream modules
+    # ------------------------------------------------------------------
+    def load_image(self, sample: RefugeSample):
+        """Return the RGB fundus image for ``sample``.
+
+        Implementors should handle color-space consistency (e.g., ensure RGB vs
+        BGR) and any global normalisation desired across devices.
+        """
+
+        raise NotImplementedError("Image loading to be implemented")
+
+    def load_mask(self, sample: RefugeSample):
+        """Return the optic disc/cup mask for ``sample`` if available."""
+
+        raise NotImplementedError("Mask loading to be implemented")
+
+    def disc_geometry(self, sample: RefugeSample) -> Dict[str, float]:
+        """Compute disc centre and diameter from the mask.
+
+        The segmentation module will rely on this to crop 2.5–3× disc-diameter
+        ROIs at training time.
+        """
+
+        raise NotImplementedError("Disc geometry helper to be implemented")
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+    def _collect_refuge1_train(self) -> List[RefugeSample]:
+        base = self.root_dir / "Train" / "REFUGE1-train"
+        if not base.exists():
+            return []
+
+        fovea_path = base / "Fovea_location.xlsx"
+        fovea_map = self._read_fovea_table(fovea_path, img_col="ImgName")
+
+        samples: List[RefugeSample] = []
+        image_root = base / "Training400"
+        mask_root = base / "Disc_Cup_Masks"
+
+        for label_name, label_val in ("Glaucoma", 1), ("Non-Glaucoma", 0):
+            img_dir = image_root / label_name
+            mask_dir = mask_root / label_name
+            if not img_dir.exists():
+                continue
+            for image_path in sorted(img_dir.glob("*.jpg")):
+                img_name = image_path.name
+                mask_path = (mask_dir / image_path.with_suffix(".bmp").name)
+                fovea = fovea_map.get(img_name)
+                sample_id = f"refuge1_train_{image_path.stem}"
+                samples.append(
+                    RefugeSample(
+                        sample_id=sample_id,
+                        dataset="refuge1",
+                        split="train",
+                        image_path=image_path,
+                        label=label_val,
+                        device=None,
+                        mask_path=mask_path if mask_path.exists() else None,
+                        fovea_coord=fovea,
+                    )
+                )
+        return samples
+
+    def _collect_refuge1_val(self) -> List[RefugeSample]:
+        base = self.root_dir / "Train" / "REFUGE1-val"
+        if not base.exists():
+            return []
+
+        fovea_path = base / "Fovea_locations.xlsx"
+        df = pd.read_excel(fovea_path)
+        samples: List[RefugeSample] = []
+        image_root = base / "REFUGE-Validation400"
+        mask_root = base / "Disc_Cup_Masks"
+
+        for _, row in df.iterrows():
+            img_name = row["ImgName"]
+            image_path = image_root / img_name
+            mask_path = mask_root / Path(img_name).with_suffix(".bmp").name
+            fovea = self._extract_fovea(row, x_key="Fovea_X", y_key="Fovea_Y")
+            label = int(row.get("Glaucoma Label", 0)) if not pd.isna(row.get("Glaucoma Label", 0)) else None
+            sample_id = f"refuge1_val_{Path(img_name).stem}"
+            samples.append(
+                RefugeSample(
+                    sample_id=sample_id,
+                    dataset="refuge1",
+                    split="val",
+                    image_path=image_path,
+                    label=label,
+                    device=None,
+                    mask_path=mask_path if mask_path.exists() else None,
+                    fovea_coord=fovea,
+                )
+            )
+        return samples
+
+    def _collect_refuge1_test(self) -> List[RefugeSample]:
+        base = self.root_dir / "Train" / "REFUGE1-test"
+        if not base.exists():
+            return []
+
+        df = pd.read_excel(base / "Glaucoma_label_and_Fovea_location.xlsx")
+        image_root = base / "Test400"
+        mask_root = base / "Disc_Cup_Masks"
+        samples: List[RefugeSample] = []
+
+        for _, row in df.iterrows():
+            img_name = row["ImgName"]
+            image_path = image_root / img_name
+            mask_path = mask_root / Path(img_name).with_suffix(".bmp").name
+            fovea = self._extract_fovea(row, x_key="Fovea_X", y_key="Fovea_Y")
+            label = int(row.get("Label(Glaucoma=1)", 0)) if not pd.isna(row.get("Label(Glaucoma=1)", 0)) else None
+            sample_id = f"refuge1_test_{Path(img_name).stem}"
+            samples.append(
+                RefugeSample(
+                    sample_id=sample_id,
+                    dataset="refuge1",
+                    split="test",
+                    image_path=image_path,
+                    label=label,
+                    device=None,
+                    mask_path=mask_path if mask_path.exists() else None,
+                    fovea_coord=fovea,
+                )
+            )
+        return samples
+
+    def _collect_refuge2_val(self) -> List[RefugeSample]:
+        base = self.root_dir / "Validation"
+        if not base.exists():
+            return []
+
+        label_df = pd.read_csv(base / "glaucoma.csv")
+        fovea_df = pd.read_csv(base / "fovea.csv")
+        fovea_map = {
+            row["ImageName"]: (float(row["Fovea_X"]), float(row["Fovea_Y"]))
+            for _, row in fovea_df.iterrows()
+        }
+        samples: List[RefugeSample] = []
+        image_root = base / "Images"
+        mask_root = base / "Disc_Masks"
+
+        for _, row in label_df.iterrows():
+            img_name = row["FileName"]
+            image_path = image_root / img_name
+            mask_path = mask_root / Path(img_name).with_suffix(".png").name
+            label = row.get("Glaucoma Risk")
+            label = int(label) if label == label else None
+            sample_id = f"refuge2_val_{Path(img_name).stem}"
+            samples.append(
+                RefugeSample(
+                    sample_id=sample_id,
+                    dataset="refuge2",
+                    split="val",
+                    image_path=image_path,
+                    label=label,
+                    device=None,
+                    mask_path=mask_path if mask_path.exists() else None,
+                    fovea_coord=fovea_map.get(img_name),
+                )
+            )
+        return samples
+
+    def _collect_refuge2_test(self) -> List[RefugeSample]:
+        base = self.root_dir / "Test"
+        if not base.exists():
+            return []
+
+        label_df = pd.read_excel(base / "task1.xls", header=None, names=["ImgName", "Glaucoma"])
+        fovea_df = pd.read_excel(base / "fovea.xlsx")
+        fovea_map = {
+            row["ImageName"]: (float(row["Fovea_X"]), float(row["Fovea_Y"]))
+            for _, row in fovea_df.iterrows()
+        }
+        samples: List[RefugeSample] = []
+        image_root = base / "refuge2-test"
+        mask_root = base / "Disc_Mask"
+
+        for _, row in label_df.iterrows():
+            img_name = row["ImgName"]
+            image_path = image_root / img_name
+            mask_path = mask_root / Path(img_name).with_suffix(".png").name
+            label = row.get("Glaucoma")
+            label = int(label) if label == label else None
+            sample_id = f"refuge2_test_{Path(img_name).stem}"
+            samples.append(
+                RefugeSample(
+                    sample_id=sample_id,
+                    dataset="refuge2",
+                    split="test",
+                    image_path=image_path,
+                    label=label,
+                    device=None,
+                    mask_path=mask_path if mask_path.exists() else None,
+            fovea_coord=fovea_map.get(img_name),
+                )
+            )
+        return samples
+
+    @staticmethod
+    def _read_fovea_table(path: Path, img_col: str) -> Dict[str, Tuple[float, float]]:
+        if not path.exists():
+            return {}
+        df = pd.read_excel(path)
+        mapping: Dict[str, Tuple[float, float]] = {}
+        for _, row in df.iterrows():
+            mapping[row[img_col]] = (
+                float(row.get("Fovea_X", float("nan"))),
+                float(row.get("Fovea_Y", float("nan"))),
+            )
+        return mapping
+
+    @staticmethod
+    def _extract_fovea(row: pd.Series, x_key: str, y_key: str) -> Optional[Tuple[float, float]]:
+        x_val = row.get(x_key)
+        y_val = row.get(y_key)
+        if pd.isna(x_val) or pd.isna(y_val):
+            return None
+        return float(x_val), float(y_val)