hypertower/scripts/grid_search_analytics/derived_statistics.py

#!/usr/bin/env python3
"""
Shared helpers for building grid search analytics.

The class below will gradually accumulate reusable utilities for working with
grid search outputs (summary.json, cli_args.json, etc).
"""

from __future__ import annotations

import json
import csv
import re
from pathlib import Path
from typing import Dict, Iterable, Iterator, List, Optional

import numpy as np

DEFAULT_EXCLUDE_KEYS = {
    "run_id",
    "fold_metrics",
    "best_metric",
    "best_metric_mode",
    "best_metric_mean",
    "best_metric_std",
    "eval_mode",
    "n_splits",
    "num_classes",
}


class GridSearchAnalytics:
    """Utility wrapper for inspecting grid search result directories."""

    def __init__(self,
                 analysis_dir: Path | str,
                 exclude_keys: Optional[Iterable[str]] = None) -> None:
        self.analysis_dir = Path(analysis_dir)
        if not self.analysis_dir.exists():
            raise FileNotFoundError(f"analysis_dir does not exist: {self.analysis_dir}")
        self.exclude_keys = set(exclude_keys or DEFAULT_EXCLUDE_KEYS)

    def iter_run_dirs(self, shallow: bool = True) -> Iterator[Path]:
        """
        Yield run directories containing grid search artifacts.

        Shallow iteration only walks direct children. Deep iteration scans the
        entire subtree.
        """
        candidates: Iterable[Path]
        if shallow:
            candidates = (p for p in sorted(self.analysis_dir.iterdir()) if p.is_dir())
        else:
            candidates = (p for p in self.analysis_dir.rglob("*") if p.is_dir())
        for run_dir in candidates:
            summary = run_dir / "summary.json"
            cli = run_dir / "cli_args.json"
            if summary.exists() or cli.exists():
                yield run_dir

    def read_summary(self, run_dir: Path) -> Optional[Dict[str, object]]:
        """Load summary.json for a run directory."""
        return self._read_json(run_dir / "summary.json")

    def read_cli_args(self, run_dir: Path) -> Optional[Dict[str, object]]:
        """Load cli_args.json for a run directory."""
        return self._read_json(run_dir / "cli_args.json")

    def read_run_id(self,
                    run_dir: Path,
                    summary: Optional[Dict[str, object]]) -> str:
        if summary:
            rid = summary.get("run_id")
            if isinstance(rid, str) and rid:
                return rid
        return run_dir.name

    def task_from_summary(self, summary: Optional[Dict[str, object]]) -> Optional[str]:
        """Infer task (binary vs multiclass) from a summary payload."""
        if not summary:
            return None
        eval_mode = summary.get("eval_mode")
        if isinstance(eval_mode, str):
            mode = eval_mode.strip().lower()
            if mode == "binary":
                return "binary"
            if mode in {"multiclass", "multi", "multi-class"}:
                return "multiclass"
        num_classes = summary.get("num_classes")
        if isinstance(num_classes, (int, float)):
            return "binary" if int(num_classes) <= 2 else "multiclass"
        return None

    def flatten_config(self,
                       data: Dict[str, object],
                       prefix: str = "",
                       exclude_keys: Optional[Iterable[str]] = None) -> Dict[str, object]:
        """Flatten nested CLI args or config dictionaries for analysis."""
        out: Dict[str, object] = {}
        excludes = set(exclude_keys or self.exclude_keys)
        for key, value in data.items():
            if key in excludes or key.startswith("best_"):
                continue
            full_key = f"{prefix}{key}" if not prefix else f"{prefix}.{key}"
            if isinstance(value, dict):
                out.update(self.flatten_config(value, full_key, exclude_keys=excludes))
                continue
            if isinstance(value, list):
                continue
            out[full_key] = value
        return out

    def fusion_correction_events(self,
                                 output_csv: Path | str | None = None,
                                 shallow: bool = True) -> Path:
        """
        Build a table of cases where the fused head is correct while both towers
        are wrong. Rows are written to CSV for downstream analysis.
        """
        output_path = Path(output_csv) if output_csv else Path("analysis_data/grid_search_analytics/fusion_corrections.csv")
        output_path.parent.mkdir(parents=True, exist_ok=True)

        rows: List[Dict[str, object]] = []
        for run_dir in self.iter_run_dirs(shallow=shallow):
            summary = self.read_summary(run_dir)
            run_id = self.read_run_id(run_dir, summary)
            folds = self._available_folds(run_dir, summary)
            for fold in folds:
                y_true = self._load_y_true(run_dir, fold)
                if y_true is None:
                    continue
                epoch_prob_paths = self._collect_epoch_prob_paths(run_dir, fold)
                if not epoch_prob_paths:
                    # Per-epoch dumps were not found; fall back to the saved fold-level probabilities.
                    base_paths = self._collect_base_prob_paths(run_dir, fold)
                    if base_paths:
                        epoch_hint = self._fold_epoch_hint(summary, fold)
                        epoch_prob_paths = {epoch_hint if epoch_hint is not None else 0: base_paths}
                for epoch, paths in epoch_prob_paths.items():
                    arrays = {head: self._load_probs_array(path) for head, path in paths.items()}
                    if not self._has_all_heads(arrays):
                        continue
                    events = self._fusion_corrections_for_probs(y_true, arrays, run_id, fold, epoch)
                    rows.extend(events)

        if rows:
            fieldnames = [
                "run_id",
                "fold",
                "epoch",
                "index",
                "y_true",
                "pred_fused",
                "pred_img",
                "pred_md",
                "conf_fused",
                "conf_img",
                "conf_md",
            ]
            with output_path.open("w", newline="") as f:
                writer = csv.DictWriter(f, fieldnames=fieldnames)
                writer.writeheader()
                writer.writerows(rows)
        else:
            output_path.write_text("")
        return output_path

    @staticmethod
    def _read_json(path: Path) -> Optional[Dict[str, object]]:
        if not path.exists():
            return None
        try:
            data = json.loads(path.read_text())
        except Exception:
            return None
        if not isinstance(data, dict):
            return None
        return data

    @staticmethod
    def _fold_epoch_hint(summary: Optional[Dict[str, object]], fold: int) -> Optional[int]:
        if not summary:
            return None
        fold_metrics = summary.get("fold_metrics") or []
        for entry in fold_metrics:
            if not isinstance(entry, dict):
                continue
            if entry.get("fold") == fold:
                stats = entry.get("stats") if isinstance(entry.get("stats"), dict) else {}
                epoch = stats.get("epoch") or entry.get("best_epoch")
                if isinstance(epoch, (int, float)):
                    return int(epoch)
        return None

    @staticmethod
    def _available_folds(run_dir: Path, summary: Optional[Dict[str, object]]) -> List[int]:
        folds: List[int] = []
        if summary:
            for entry in summary.get("fold_metrics") or []:
                if not isinstance(entry, dict):
                    continue
                fold_idx = entry.get("fold")
                if isinstance(fold_idx, int):
                    folds.append(fold_idx)
        if not folds:
            pattern = re.compile(r"fold(\d+)_y_true\.npy$")
            for path in run_dir.glob("fold*_y_true.npy"):
                match = pattern.match(path.name)
                if match:
                    folds.append(int(match.group(1)))
        return sorted(set(folds))

    @staticmethod
    def _load_y_true(run_dir: Path, fold: int) -> Optional[np.ndarray]:
        path = run_dir / f"fold{fold}_y_true.npy"
        if not path.exists():
            return None
        try:
            return np.load(path)
        except Exception:
            return None

    @staticmethod
    def _collect_epoch_prob_paths(run_dir: Path, fold: int) -> Dict[int, Dict[str, Path]]:
        pattern = re.compile(rf"fold{fold}_epoch(\d+)_probs_(\w+)\.npy$")
        epoch_paths: Dict[int, Dict[str, Path]] = {}
        for path in run_dir.glob(f"fold{fold}_epoch*_probs_*.npy"):
            match = pattern.match(path.name)
            if not match:
                continue
            epoch = int(match.group(1))
            head = match.group(2)
            epoch_paths.setdefault(epoch, {})[head] = path
        return epoch_paths

    @staticmethod
    def _collect_base_prob_paths(run_dir: Path, fold: int) -> Dict[str, Path]:
        paths: Dict[str, Path] = {}
        for head in ("fused", "img", "md"):
            candidate = run_dir / f"fold{fold}_probs_{head}.npy"
            if candidate.exists():
                paths[head] = candidate
        return paths

    @staticmethod
    def _load_probs_array(path: Path) -> Optional[np.ndarray]:
        try:
            return np.load(path)
        except Exception:
            return None

    @staticmethod
    def _prepare_probs(arr: np.ndarray) -> Optional[np.ndarray]:
        if arr is None:
            return None
        probs = np.asarray(arr, dtype=float)
        if probs.ndim == 1:
            probs = np.stack([1.0 - probs, probs], axis=1)
        if probs.ndim != 2:
            return None
        return probs

    @staticmethod
    def _has_all_heads(arrays: Dict[str, Optional[np.ndarray]]) -> bool:
        needed = ("fused", "img", "md")
        return all(arrays.get(head) is not None for head in needed)

    def _fusion_corrections_for_probs(self,
                                      y_true: np.ndarray,
                                      arrays: Dict[str, np.ndarray],
                                      run_id: str,
                                      fold: int,
                                      epoch: int) -> List[Dict[str, object]]:
        fused = self._prepare_probs(arrays.get("fused"))
        img = self._prepare_probs(arrays.get("img"))
        md = self._prepare_probs(arrays.get("md"))
        if fused is None or img is None or md is None:
            return []
        if not (len(fused) == len(img) == len(md) == len(y_true)):
            return []

        fused_pred = fused.argmax(axis=1)
        img_pred = img.argmax(axis=1)
        md_pred = md.argmax(axis=1)

        fused_conf = np.take_along_axis(fused, fused_pred[:, None], axis=1).squeeze(1)
        img_conf = np.take_along_axis(img, img_pred[:, None], axis=1).squeeze(1)
        md_conf = np.take_along_axis(md, md_pred[:, None], axis=1).squeeze(1)

        mask = (fused_pred == y_true) & (img_pred != y_true) & (md_pred != y_true)
        indices = np.nonzero(mask)[0]

        events: List[Dict[str, object]] = []
        for idx in indices:
            events.append({
                "run_id": run_id,
                "fold": fold,
                "epoch": epoch,
                "index": int(idx),
                "y_true": int(y_true[idx]),
                "pred_fused": int(fused_pred[idx]),
                "pred_img": int(img_pred[idx]),
                "pred_md": int(md_pred[idx]),
                "conf_fused": float(fused_conf[idx]),
                "conf_img": float(img_conf[idx]),
                "conf_md": float(md_conf[idx]),
            })
        return events


if __name__ == "__main__":
    analytics = GridSearchAnalytics(Path("analysis_data/grid_search"))
    output = analytics.fusion_correction_events()
    print(f"Fusion correction events written to {output}")