#!/usr/bin/env python3 """Derived analysis wrapper for grid search analytics.""" from __future__ import annotations import json import math import re from pathlib import Path from typing import Dict, Iterable, Iterator, List, Optional import numpy as np import pandas as pd import matplotlib.pyplot as plt from scipy import stats from tqdm import tqdm class derived_analysis: def __init__( self, analysis_dir: Path | str, exclude_keys: Optional[Iterable[str]] = None, classification_mode: str = "binary", ) -> None: self.analysis_dir = Path(analysis_dir) if not self.analysis_dir.exists(): raise FileNotFoundError(f"analysis_dir does not exist: {self.analysis_dir}") mode = str(classification_mode).strip().lower() if mode not in {"binary", "multiclass"}: raise ValueError( f"classification_mode must be 'binary' or 'multiclass' (got {classification_mode!r})" ) self.classification_mode = mode self.exclude_keys = set(exclude_keys or []) self.fusion_corrections = pd.DataFrame() self.fusion_errors = pd.DataFrame() self.statistics_df = pd.DataFrame() self.primary_metrics = pd.DataFrame() self.fusion_performance_corr = pd.DataFrame() self.param_perf_corr = pd.DataFrame() self.se_mode_effects = {} def identify_fusion_corrections( self, shallow: bool = True, existing: bool = True ) -> pd.DataFrame: cache_path = self._fusion_corrections_path() if existing and cache_path.exists(): df = self._read_fusion_corrections(cache_path) self.fusion_corrections = df errors_path = self._fusion_errors_path() if errors_path.exists(): self.fusion_errors = self._read_fusion_errors(errors_path) else: self.fusion_errors = pd.DataFrame() self.statistics_df = self._build_statistics_df(df, shallow=shallow) return self.fusion_corrections rows: List[Dict[str, object]] = [] error_rows: List[Dict[str, object]] = [] stats_rows: List[Dict[str, object]] = [] for run_dir in self._iter_run_dirs(shallow=shallow, show_progress=True): summary = self._read_summary(run_dir) cli = self._read_cli_args(run_dir) mode = self._infer_mode(summary, cli) if mode != self.classification_mode: continue run_id = self._read_run_id(run_dir, summary) folds = self._available_folds(run_dir, summary) run_count = 0 grid_params = self._grid_params_from_cli(cli, summary) for fold in folds: y_true = self._load_y_true(run_dir, fold) if y_true is None: continue epoch_prob_paths = self._collect_epoch_prob_paths(run_dir, fold) if not epoch_prob_paths: base_paths = self._collect_base_prob_paths(run_dir, fold) if base_paths: epoch_hint = self._fold_epoch_hint(summary, fold) epoch_prob_paths = { epoch_hint if epoch_hint is not None else 0: base_paths } for epoch, paths in epoch_prob_paths.items(): arrays = { head: self._load_probs_array(path) for head, path in paths.items() } if not self._has_all_heads(arrays): continue events = self._fusion_corrections_for_probs( y_true, arrays, run_id, fold, epoch ) run_count += len(events) rows.extend(events) errors = self._fusion_errors_for_probs( y_true, arrays, run_id, fold, epoch ) error_rows.extend(errors) stats_rows.append( { "run_id": run_id, "run_dir": str(run_dir), "classification_mode": mode, "fusion_corrections": int(run_count), **grid_params, } ) self.fusion_corrections = pd.DataFrame(rows) self.fusion_errors = pd.DataFrame(error_rows) self.statistics_df = pd.DataFrame(stats_rows) return self.fusion_corrections def write_fusion_corrections(self, output_path: Path | str | None = None) -> Path: if self.fusion_corrections.empty: self.identify_fusion_corrections(existing=True) path = Path(output_path) if output_path else self._fusion_corrections_path() path.parent.mkdir(parents=True, exist_ok=True) self.fusion_corrections.to_csv(path, index=False) return path def write_fusion_errors(self, output_path: Path | str | None = None) -> Path: if self.fusion_errors.empty: self.identify_fusion_corrections(existing=True) path = Path(output_path) if output_path else self._fusion_errors_path() path.parent.mkdir(parents=True, exist_ok=True) self.fusion_errors.to_csv(path, index=False) return path def populate_primary_metrics( self, shallow: bool = True, show_progress: bool = True, existing: bool = True ) -> pd.DataFrame: cache_path = self._primary_metrics_path() if existing and cache_path.exists(): df = self._read_primary_metrics(cache_path) self.primary_metrics = df summary_df = self._aggregate_primary_metrics(df) if summary_df.empty: if self.statistics_df.empty: self.statistics_df = summary_df else: if self.statistics_df.empty: self.statistics_df = summary_df else: self.statistics_df = self.statistics_df.merge( summary_df, on=["run_id", "run_dir", "classification_mode"], how="left", ) return self.primary_metrics rows: List[Dict[str, object]] = [] for run_dir in self._iter_run_dirs( shallow=shallow, show_progress=show_progress ): summary = self._read_summary(run_dir) cli = self._read_cli_args(run_dir) mode = self._infer_mode(summary, cli) if mode != self.classification_mode: continue run_id = self._read_run_id(run_dir, summary) folds = self._available_folds(run_dir, summary) if not folds: continue for fold in folds: log_df = self._read_epoch_log(run_dir, fold) best_epoch, holdout_best_epoch = self._extract_best_epochs(log_df) if best_epoch is None or holdout_best_epoch is None: continue row: Dict[str, object] = { "run_id": run_id, "run_dir": str(run_dir), "classification_mode": mode, "fold": int(fold), "best_epoch": int(best_epoch), "holdout_best_epoch": int(holdout_best_epoch), } for head_key, log_suffix, roc_suffix in ( ("fused", "fused", "fused"), ("image", "img", "image"), ("metadata", "md", "metadata"), ): best_auc = self._load_auc_for_epoch( run_dir, fold, best_epoch, roc_suffix, holdout=False ) hold_auc = self._load_auc_for_epoch( run_dir, fold, holdout_best_epoch, roc_suffix, holdout=True ) row[f"best_auc_{head_key}"] = best_auc row[f"holdout_best_auc_{head_key}"] = hold_auc if log_df is not None: best_row = self._row_for_epoch(log_df, best_epoch) hold_row = self._row_for_epoch(log_df, holdout_best_epoch) best_acc = self._metric_from_row(best_row, f"acc_{log_suffix}") hold_acc = self._metric_from_row( hold_row, f"holdout_acc_{log_suffix}" ) row[f"best_acc_{head_key}"] = best_acc row[f"holdout_best_acc_{head_key}"] = hold_acc rows.append(row) self.primary_metrics = pd.DataFrame(rows) summary_df = self._aggregate_primary_metrics(self.primary_metrics) if summary_df.empty: if self.statistics_df.empty: self.statistics_df = summary_df else: if self.statistics_df.empty: self.statistics_df = summary_df else: self.statistics_df = self.statistics_df.merge( summary_df, on=["run_id", "run_dir", "classification_mode"], how="left", ) return self.primary_metrics def write_primary_metrics(self, output_path: Path | str | None = None) -> Path: if self.primary_metrics.empty: self.populate_primary_metrics() path = Path(output_path) if output_path else self._primary_metrics_path() path.parent.mkdir(parents=True, exist_ok=True) self.primary_metrics.to_csv(path, index=False) return path def plot_fusion_corrections_errors( self, output_path: Path | str | None = None, shallow: bool = True, existing: bool = True, top_n: int | None = None, ) -> pd.DataFrame: if self.fusion_corrections.empty: self.identify_fusion_corrections(shallow=shallow, existing=existing) if self.fusion_corrections.empty: raise RuntimeError( "fusion_corrections is empty; run identify_fusion_corrections() first." ) corrections = ( self.fusion_corrections.groupby("run_id") .size() .rename("fusion_corrections") ) if self.fusion_errors.empty: errors = corrections.copy() * 0 errors.name = "fusion_errors" else: errors = self.fusion_errors.groupby("run_id").size().rename("fusion_errors") df = pd.concat([corrections, errors], axis=1).fillna(0).reset_index() df = df.sort_values(by="run_id") if top_n is not None: df = df.head(int(top_n)) out_path = ( Path(output_path) if output_path else ( self.analysis_dir / "plots" / f"fusion_corrections_errors_{self.classification_mode}.png" ) ) out_path.parent.mkdir(parents=True, exist_ok=True) fig, ax = plt.subplots(figsize=(10, 4.8)) x = np.arange(len(df)) ax.bar( x, df["fusion_corrections"], color="steelblue", width=1.0, label="fusion_corrections", ) ax.bar( x, df["fusion_errors"], bottom=df["fusion_corrections"], color="tomato", width=1.0, label="fusion_errors", ) ax.set_xticks([]) ax.set_ylabel("Count") ax.set_title( f"Fusion corrections + errors per run ({self.classification_mode})" ) ax.legend(loc="upper right") ax.grid(True, axis="y", alpha=0.3, linestyle="--") fig.tight_layout() fig.savefig(out_path, dpi=170) plt.close(fig) return df def plot_conf_delta_boxplot( self, output_path: Path | str | None = None, shallow: bool = True, existing: bool = True, top_n: int | None = None, ) -> pd.DataFrame: if self.fusion_corrections.empty: self.identify_fusion_corrections(shallow=shallow, existing=existing) if self.fusion_corrections.empty: raise RuntimeError( "fusion_corrections is empty; run identify_fusion_corrections() first." ) df = self.fusion_corrections.copy() df["conf_delta"] = df["conf_fused"] - 0.5 * (df["conf_img"] + df["conf_md"]) mean_order = df.groupby("run_id")["conf_delta"].mean().sort_values() run_order = mean_order.index.tolist() if top_n is not None: run_order = run_order[: int(top_n)] data = [ df.loc[df["run_id"] == run_id, "conf_delta"].values for run_id in run_order ] out_path = ( Path(output_path) if output_path else ( self.analysis_dir / "plots" / f"conf_delta_box_{self.classification_mode}.png" ) ) out_path.parent.mkdir(parents=True, exist_ok=True) fig, ax = plt.subplots(figsize=(10, 4.8)) ax.boxplot(data, widths=0.6, showfliers=False) ax.set_xticks([]) ax.set_ylabel("conf_delta (fused - mean(towers))") ax.set_title(f"Confidence delta per run ({self.classification_mode})") ax.grid(True, axis="y", alpha=0.3, linestyle="--") fig.tight_layout() fig.savefig(out_path, dpi=170) plt.close(fig) return df def param_performance_correlations( self, output_path: Path | str | None = None, shallow: bool = True, existing: bool = True, method: str = "spearman", cat_method: str = "kruskal", ) -> pd.DataFrame: if self.statistics_df.empty: self.identify_fusion_corrections(shallow=shallow, existing=existing) if self.primary_metrics.empty: self.populate_primary_metrics(shallow=shallow, existing=existing) df = self.statistics_df.copy() if df.empty: raise RuntimeError( "statistics_df is empty; run identify_fusion_corrections() first." ) if self.fusion_corrections.empty: self.identify_fusion_corrections(shallow=shallow, existing=existing) conf_delta = None if not self.fusion_corrections.empty: fc = self.fusion_corrections.copy() fc["conf_delta"] = fc["conf_fused"] - 0.5 * (fc["conf_img"] + fc["conf_md"]) conf_delta = ( fc.groupby("run_id")["conf_delta"].mean().rename("conf_delta_mean") ) df = df.merge(conf_delta.reset_index(), on="run_id", how="left") if self.fusion_errors.empty: errors_path = self._fusion_errors_path() if errors_path.exists(): self.fusion_errors = self._read_fusion_errors(errors_path) if not self.fusion_errors.empty and "fusion_corrections" in df.columns: err_counts = ( self.fusion_errors.groupby("run_id").size().rename("fusion_errors") ) df = df.merge(err_counts.reset_index(), on="run_id", how="left") df["fusion_errors"] = df["fusion_errors"].fillna(0) eps = 1e-6 df["error_correction_ratio"] = (df["fusion_errors"] + eps) / ( df["fusion_corrections"] + eps ) metric_cols = [] for cand in ("holdout_best_acc_fused_mean", "best_acc_fused_mean"): if cand in df.columns: metric_cols.append(("acc", cand)) break for cand in ("holdout_best_auc_fused_mean", "best_auc_fused_mean"): if cand in df.columns: metric_cols.append(("auc", cand)) break if "fusion_corrections" in df.columns: metric_cols.append(("fusion_corrections", "fusion_corrections")) if conf_delta is not None and "conf_delta_mean" in df.columns: metric_cols.append(("conf_delta", "conf_delta_mean")) if "error_correction_ratio" in df.columns: metric_cols.append(("error_correction_ratio", "error_correction_ratio")) if not metric_cols: raise RuntimeError("No metrics found in statistics_df for correlation.") grid_param_keys = [ "crop_variant", "crop_normalize", "crop_weights", "crop_tta", "loss_mode", "thaw_mode", "se_mode", "se_bridge_pre_norm", "se_tower_pre_norm", ] param_cols = [c for c in grid_param_keys if c in df.columns] def _to_float(v): if v is None: return None if isinstance(v, bool): return None if isinstance(v, (int, float)) and not math.isnan(float(v)): return float(v) try: return float(v) except Exception: return None def _format_value(v: object) -> str: if v is None: return "" if isinstance(v, bool): return "true" if v else "false" if isinstance(v, int): return str(v) if isinstance(v, float): return f"{v:.6g}" return str(v) def _rankdata(vals: List[float]) -> List[float]: order = sorted(range(len(vals)), key=lambda i: vals[i]) ranks = [0.0] * len(vals) i = 0 while i < len(vals): j = i while j + 1 < len(vals) and vals[order[j + 1]] == vals[order[i]]: j += 1 avg_rank = (i + j) / 2.0 + 1.0 for k in range(i, j + 1): ranks[order[k]] = avg_rank i = j + 1 return ranks def _pearson(x: List[float], y: List[float]) -> Optional[float]: if len(x) < 2: return None mx = sum(x) / len(x) my = sum(y) / len(y) num = sum((xi - mx) * (yi - my) for xi, yi in zip(x, y)) denx = sum((xi - mx) ** 2 for xi in x) deny = sum((yi - my) ** 2 for yi in y) if denx <= 0 or deny <= 0: return None return num / math.sqrt(denx * deny) def _spearman(x: List[float], y: List[float]) -> Optional[float]: return _pearson(_rankdata(x), _rankdata(y)) def _eta(categories: List[object], values: List[float]) -> Optional[float]: if len(values) < 2: return None overall = sum(values) / len(values) total = sum((v - overall) ** 2 for v in values) if total <= 0: return None groups = {} for cat, val in zip(categories, values): groups.setdefault(cat, []).append(val) between = 0.0 for vals in groups.values(): avg = sum(vals) / len(vals) between += len(vals) * (avg - overall) ** 2 return math.sqrt(between / total) cat_method_norm = cat_method.strip().lower() if cat_method_norm not in {"eta", "anova", "kruskal"}: raise ValueError( f"cat_method must be 'eta', 'anova', or 'kruskal' (got {cat_method!r})" ) rows: List[Dict[str, object]] = [] for name, metric_col in metric_cols: metric_vals = df[metric_col] for param in param_cols: param_vals = df[param] pairs = [ (p, m) for p, m in zip(param_vals, metric_vals) if m is not None and not (isinstance(m, float) and math.isnan(m)) ] if len(pairs) < 3: continue p_vals, m_vals = zip(*pairs) group_means: Dict[object, float] = {} for p, m in pairs: group_means.setdefault(p, []).append(m) group_means = { k: float(sum(v) / len(v)) for k, v in group_means.items() } if name == "error_correction_ratio": best_value = min(group_means.items(), key=lambda item: item[1])[0] else: best_value = max(group_means.items(), key=lambda item: item[1])[0] num_vals = [] numeric_ok = True for v in p_vals: num = _to_float(v) if num is None: numeric_ok = False break num_vals.append(num) if numeric_ok and len(set(num_vals)) >= 3: p_val = None if method == "spearman": try: corr, p_val = stats.spearmanr(num_vals, list(m_vals)) except Exception: corr = None else: try: corr, p_val = stats.pearsonr(num_vals, list(m_vals)) except Exception: corr = None if corr is not None and corr != corr: corr = None rows.append( { "metric": name, "metric_col": metric_col, "param": param, "type": "numeric", "n": len(pairs), "corr": corr, "stat": corr, "p_value": p_val, "method": method, "best": _format_value(best_value), } ) else: stat_val = None p_val = None corr = None groups: Dict[object, List[float]] = {} for p, m in pairs: groups.setdefault(p, []).append(m) group_vals = [vals for vals in groups.values() if len(vals) > 0] if cat_method_norm == "eta": corr = _eta(list(p_vals), list(m_vals)) stat_val = corr elif cat_method_norm == "anova": if len(group_vals) >= 2: try: stat_val, p_val = stats.f_oneway(*group_vals) corr = stat_val except Exception: stat_val = None elif cat_method_norm == "kruskal": if len(group_vals) >= 2: try: stat_val, p_val = stats.kruskal(*group_vals) corr = stat_val except Exception: stat_val = None rows.append( { "metric": name, "metric_col": metric_col, "param": param, "type": "categorical", "n": len(pairs), "corr": corr, "stat": stat_val, "p_value": p_val, "method": cat_method_norm, "best": _format_value(best_value), } ) out_df = pd.DataFrame(rows).sort_values( by=["metric", "corr"], ascending=[True, False] ) self.param_perf_corr = out_df out_path = ( Path(output_path) if output_path else ( self.analysis_dir / "plots" / f"param_perf_corr_{self.classification_mode}.csv" ) ) out_path.parent.mkdir(parents=True, exist_ok=True) out_df.to_csv(out_path, index=False) return out_df def plot_param_perf_corr_panels( self, corr_df: pd.DataFrame, output_path: Path | str | None = None, ) -> pd.DataFrame: if corr_df.empty: raise RuntimeError( "corr_df is empty; run param_performance_correlations() first." ) metrics = ["auc", "acc", "fusion_corrections", "conf_delta"] auc_df = corr_df[corr_df["metric"] == "auc"].copy() if auc_df.empty: raise RuntimeError("No 'auc' metric rows found in corr_df.") auc_df = auc_df.sort_values(by="corr", ascending=False) order = auc_df["param"].tolist() out_path = ( Path(output_path) if output_path else ( self.analysis_dir / "plots" / f"param_perf_corr_panels_{self.classification_mode}.png" ) ) out_path.parent.mkdir(parents=True, exist_ok=True) fig, axes = plt.subplots(2, 2, figsize=(12, 8), sharey=True) axes = axes.flatten() for idx, metric in enumerate(metrics): ax = axes[idx] sub = corr_df[corr_df["metric"] == metric].set_index("param") sub = sub.reindex(order) values = sub["corr"].astype(float).values y = np.arange(len(order)) ax.barh(y, values, color="steelblue") ax.axvline(0.0, color="black", lw=1) ax.set_title(metric) ax.set_yticks(y) ax.set_yticklabels(order, fontsize=7) ax.grid(True, axis="x", alpha=0.3, linestyle="--") # annotate p-values when available for i, param in enumerate(order): if param not in sub.index: continue p_val = sub.loc[param, "p_value"] if p_val is None or (isinstance(p_val, float) and np.isnan(p_val)): continue ax.text( values[i] if not np.isnan(values[i]) else 0.0, i, f" p={p_val:.3g}", va="center", ha="left" if values[i] >= 0 else "right", fontsize=7, ) # print p-values to console for each metric print(f"\n[{metric}] p-values") for param in order: if param not in sub.index: continue p_val = sub.loc[param, "p_value"] if p_val is None or (isinstance(p_val, float) and np.isnan(p_val)): continue print(f" {param}: p={p_val:.4g}") fig.suptitle("Parameter correlations (ordered by AUC correlation)", fontsize=12) fig.tight_layout(rect=[0, 0.02, 1, 0.96]) fig.savefig(out_path, dpi=170) plt.close(fig) return corr_df def se_mode_effects_summary( self, metric: str = "auc", metric_col: str | None = None, head: str = "fused", prefer_holdout: bool = True, top_n: int | None = None, top_metric_col: str | None = None, output_dir: Path | str | None = None, pairwise_method: str = "mannwhitney", shallow: bool = True, existing: bool = True, ) -> Dict[str, pd.DataFrame]: if self.statistics_df.empty: self.identify_fusion_corrections(shallow=shallow, existing=existing) if self.primary_metrics.empty: self.populate_primary_metrics(shallow=shallow, existing=existing) df = self.statistics_df.copy() if df.empty: raise RuntimeError( "statistics_df is empty; run identify_fusion_corrections() first." ) metric_norm = metric.strip().lower() if metric_norm not in {"acc", "auc"}: raise ValueError(f"metric must be 'acc' or 'auc' (got {metric!r})") if metric_col is None: candidates = [] if prefer_holdout: candidates.append(f"holdout_best_{metric_norm}_{head}_mean") candidates.append(f"best_{metric_norm}_{head}_mean") else: candidates.append(f"best_{metric_norm}_{head}_mean") candidates.append(f"holdout_best_{metric_norm}_{head}_mean") for cand in candidates: if cand in df.columns: metric_col = cand break if metric_col is None or metric_col not in df.columns: raise RuntimeError( "Could not find a metric column to summarize; run populate_primary_metrics() " "or pass metric_col explicitly." ) if "se_mode" not in df.columns: raise RuntimeError("statistics_df is missing se_mode column.") use_cols = ["run_id", "se_mode", metric_col] if "se_bridge_pre_norm" in df.columns: use_cols.append("se_bridge_pre_norm") if "se_tower_pre_norm" in df.columns: use_cols.append("se_tower_pre_norm") df = df[use_cols].copy() df = df.dropna(subset=[metric_col, "se_mode"]) if df.empty: raise RuntimeError( "No rows available after filtering for se_mode and metric." ) if top_n is not None: top_metric = top_metric_col or metric_col if top_metric not in df.columns: raise RuntimeError( f"top_metric_col {top_metric!r} not found in statistics_df." ) df = df.sort_values(by=top_metric, ascending=False).head(int(top_n)) if df.empty: raise RuntimeError("No rows available after applying top_n filter.") summary = ( df.groupby("se_mode")[metric_col] .agg(["count", "mean", "median", "std"]) .reset_index() .rename(columns={"count": "n"}) ) # Pairwise comparisons pairwise_rows: List[Dict[str, object]] = [] modes = summary["se_mode"].tolist() pairwise_method_norm = pairwise_method.strip().lower() if pairwise_method_norm not in {"mannwhitney", "ttest"}: raise ValueError( f"pairwise_method must be 'mannwhitney' or 'ttest' (got {pairwise_method!r})" ) def _cohens_d(a: np.ndarray, b: np.ndarray) -> float: if len(a) < 2 or len(b) < 2: return float("nan") va = np.var(a, ddof=1) vb = np.var(b, ddof=1) pooled = ((len(a) - 1) * va + (len(b) - 1) * vb) / max( len(a) + len(b) - 2, 1 ) if pooled <= 0: return float("nan") return (np.mean(a) - np.mean(b)) / math.sqrt(pooled) for i, m1 in enumerate(modes): vals1 = df.loc[df["se_mode"] == m1, metric_col].astype(float).values if vals1.size == 0: continue for m2 in modes[i + 1 :]: vals2 = df.loc[df["se_mode"] == m2, metric_col].astype(float).values if vals2.size == 0: continue p_val = None stat_val = None if pairwise_method_norm == "mannwhitney": try: stat_val, p_val = stats.mannwhitneyu( vals1, vals2, alternative="two-sided" ) except Exception: stat_val, p_val = None, None else: try: stat_val, p_val = stats.ttest_ind(vals1, vals2, equal_var=False) except Exception: stat_val, p_val = None, None pairwise_rows.append( { "metric_col": metric_col, "se_mode_a": m1, "se_mode_b": m2, "n_a": int(vals1.size), "n_b": int(vals2.size), "mean_a": float(np.mean(vals1)), "mean_b": float(np.mean(vals2)), "mean_diff": float(np.mean(vals1) - np.mean(vals2)), "median_a": float(np.median(vals1)), "median_b": float(np.median(vals2)), "median_diff": float(np.median(vals1) - np.median(vals2)), "cohens_d": _cohens_d(vals1, vals2), "stat": stat_val, "p_value": p_val, "method": pairwise_method_norm, } ) pairwise_df = pd.DataFrame(pairwise_rows) # Stratified by pre-norm options (within relevant se_mode) bridge_df = pd.DataFrame() if "se_bridge_pre_norm" in df.columns: bridge_df = ( df[df["se_mode"].isin(["bridge", "both"])] .groupby(["se_mode", "se_bridge_pre_norm"])[metric_col] .agg(["count", "mean", "median", "std"]) .reset_index() .rename(columns={"count": "n"}) ) tower_df = pd.DataFrame() if "se_tower_pre_norm" in df.columns: tower_df = ( df[df["se_mode"].isin(["tower", "both"])] .groupby(["se_mode", "se_tower_pre_norm"])[metric_col] .agg(["count", "mean", "median", "std"]) .reset_index() .rename(columns={"count": "n"}) ) result = { "summary": summary, "pairwise": pairwise_df, "bridge_pre_norm": bridge_df, "tower_pre_norm": tower_df, } self.se_mode_effects = result if output_dir is not None: out_dir = Path(output_dir) else: out_dir = self.analysis_dir / "plots" out_dir.mkdir(parents=True, exist_ok=True) summary.to_csv(out_dir / f"se_mode_summary_{metric_col}.csv", index=False) if not pairwise_df.empty: pairwise_df.to_csv( out_dir / f"se_mode_pairwise_{metric_col}.csv", index=False ) if not bridge_df.empty: bridge_df.to_csv( out_dir / f"se_mode_bridge_pre_norm_{metric_col}.csv", index=False ) if not tower_df.empty: tower_df.to_csv( out_dir / f"se_mode_tower_pre_norm_{metric_col}.csv", index=False ) return result def fusion_corrections_correlation( self, output_path: Path | str | None = None, method: str = "pearson", metric_type: str = "acc", ) -> pd.DataFrame: if self.fusion_corrections.empty: raise RuntimeError( "fusion_corrections is empty; run identify_fusion_corrections() first." ) if self.primary_metrics.empty: raise RuntimeError( "primary_metrics is empty; run populate_primary_metrics() first." ) if "fold" not in self.primary_metrics.columns: raise RuntimeError( "primary_metrics missing fold column; refresh populate_primary_metrics()." ) method_norm = method.strip().lower() if method_norm not in {"pearson", "spearman"}: raise ValueError(f"method must be 'pearson' or 'spearman' (got {method!r})") metric_norm = metric_type.strip().lower() if metric_norm not in {"acc", "auc"}: raise ValueError( f"metric_type must be 'acc' or 'auc' (got {metric_type!r})" ) metric_cols = [ c for c in self.primary_metrics.columns if c.startswith(("best_", "holdout_best_")) and f"_{metric_norm}_" in c and not c.endswith(("_mean", "_sd")) ] if not metric_cols: raise RuntimeError( "No primary metric columns found in primary_metrics; run populate_primary_metrics() first." ) fold_counts = self._fold_sample_and_opportunity_counts(self.primary_metrics) best_epochs = self.primary_metrics[ ["run_id", "run_dir", "fold", "best_epoch"] ].dropna() warmup_map = self._build_warmup_map(best_epochs) best_epochs = best_epochs.merge(warmup_map, on="run_id", how="left") best_epochs["warmup_end"] = best_epochs["warmup_end"].fillna(0).astype(int) best_epochs["best_epoch"] = best_epochs["best_epoch"].astype(int) events = self.fusion_corrections.merge( best_epochs[["run_id", "fold", "best_epoch", "warmup_end"]], on=["run_id", "fold"], how="inner", ) if "epoch" in events.columns: events = events[ (events["epoch"] >= events["warmup_end"]) & (events["epoch"] <= events["best_epoch"]) ] counts = ( events.groupby(["run_id", "fold"], as_index=False) .size() .rename(columns={"size": "fusion_corrections"}) ) merged = self.primary_metrics.merge(counts, on=["run_id", "fold"], how="left") merged = merged.merge(fold_counts, on=["run_id", "run_dir", "fold"], how="left") merged["fusion_corrections"] = merged["fusion_corrections"].fillna(0) merged["n_samples"] = merged["n_samples"].replace(0, np.nan) merged["both_wrong"] = merged["both_wrong"].replace(0, np.nan) merged["fusion_corrections_rate"] = ( merged["fusion_corrections"] / merged["n_samples"] ) merged["fusion_corrections_per_opportunity"] = ( merged["fusion_corrections"] / merged["both_wrong"] ) merged = self._add_fusion_gain_columns(merged) gain_cols = [ c for c in merged.columns if c.endswith("_fusion_gain") and c.startswith(("best_", "holdout_best_")) and f"_{metric_norm}_" in c ] error_counts = self._fusion_errors_counts(merged) merged = merged.merge(error_counts, on=["run_id", "fold"], how="left") merged["fusion_errors"] = merged["fusion_errors"].fillna(0) eps = 1e-6 merged["correction_error_rate"] = (merged["fusion_corrections"] + eps) / ( merged["fusion_errors"] + eps ) rows = [] x_metrics = [ "fusion_corrections", "fusion_corrections_rate", "fusion_corrections_per_opportunity", "fusion_errors", "correction_error_rate", ] all_metrics = metric_cols + gain_cols for x in x_metrics: if x not in merged.columns: continue for col in all_metrics: sub = merged[[x, col]].dropna() if len(sub) < 2: corr = np.nan else: corr = float(sub[x].corr(sub[col], method=method_norm)) rows.append( { "x_metric": x, "metric": col, "corr": corr, "n": int(len(sub)), "metric_type": metric_norm, } ) out_df = pd.DataFrame(rows).sort_values( by=["x_metric", "corr"], ascending=[True, False] ) self.fusion_performance_corr = out_df plot_x = "fusion_corrections_per_opportunity" if plot_x not in out_df["x_metric"].unique(): plot_x = "fusion_corrections" plot_df = out_df[out_df["x_metric"] == plot_x] path = ( Path(output_path) if output_path else self._fusion_performance_corr_path(method_norm, metric_norm) ) path.parent.mkdir(parents=True, exist_ok=True) self._plot_correlation_bars( plot_df, path, method=method_norm, x_metric=plot_x, metric_type=metric_norm ) return out_df def plot_fusion_perf_summary( self, corr_acc: pd.DataFrame, corr_auc: pd.DataFrame, output_path: Path | str | None = None, method: str = "spearman", x_metric: str = "fusion_corrections", ) -> pd.DataFrame: keep_templates = [ "best_acc_fused", "holdout_best_acc_fused", "best_acc_fusion_gain", "holdout_best_acc_fusion_gain", "best_auc_fused", "holdout_best_auc_fused", "best_auc_fusion_gain", "holdout_best_auc_fusion_gain", ] def _select(df: pd.DataFrame) -> pd.DataFrame: if df.empty: return df sub = df[df["x_metric"] == x_metric].copy() sub = sub[sub["metric"].isin(keep_templates)] sub = sub.drop_duplicates(subset=["metric"]) sub["metric"] = sub["metric"].str.replace("_fused", "", regex=False) return sub acc_df = _select(corr_acc) auc_df = _select(corr_auc) merged = pd.concat([acc_df, auc_df], axis=0, ignore_index=True) if merged.empty: raise RuntimeError("No matching rows found in corr_acc/corr_auc.") merged = ( merged.set_index("metric") .loc[[m.replace("_fused", "") for m in keep_templates]] .reset_index() ) safe_x = x_metric.replace("fusion_", "") out_path = ( Path(output_path) if output_path else ( self.analysis_dir / "plots" / f"fusion_{safe_x}_vs_performance_{method}.png" ) ) out_path.parent.mkdir(parents=True, exist_ok=True) self._plot_correlation_bars( merged, out_path, method=method, x_metric=x_metric, metric_type="acc/auc", title=f"Fusion corrections vs performance ({method})", ) return merged def _iter_run_dirs( self, shallow: bool = True, show_progress: bool = False ) -> Iterator[Path]: if shallow: candidates = [p for p in sorted(self.analysis_dir.iterdir()) if p.is_dir()] else: candidates = [p for p in self.analysis_dir.rglob("*") if p.is_dir()] iterator = ( tqdm(candidates, desc="Scanning runs", unit="run", leave=False) if show_progress else candidates ) for run_dir in iterator: summary = run_dir / "summary.json" cli = run_dir / "cli_args.json" if summary.exists() or cli.exists(): yield run_dir def _fusion_corrections_path(self) -> Path: fname = f"fusion_corrections_{self.classification_mode}.csv" return self.analysis_dir / fname def _fusion_errors_path(self) -> Path: fname = f"fusion_errors_{self.classification_mode}.csv" return self.analysis_dir / fname @staticmethod def _read_fusion_errors(path: Path) -> pd.DataFrame: try: df = pd.read_csv(path) except Exception: return pd.DataFrame() return df def _primary_metrics_path(self) -> Path: fname = f"primary_metrics_{self.classification_mode}.csv" return self.analysis_dir / fname def _fusion_performance_corr_path(self, method: str, metric_type: str) -> Path: fname = f"fusion_performance_corr_{self.classification_mode}_{metric_type}_{method}.png" return self.analysis_dir / fname @staticmethod def _read_primary_metrics(path: Path) -> pd.DataFrame: try: return pd.read_csv(path) except Exception: return pd.DataFrame() @staticmethod def _read_fusion_corrections(path: Path) -> pd.DataFrame: try: df = pd.read_csv(path) except Exception: return pd.DataFrame() return df def _build_statistics_df( self, df: pd.DataFrame, shallow: bool = True ) -> pd.DataFrame: counts = {} if not df.empty and "run_id" in df.columns: counts = df.groupby("run_id").size().to_dict() rows: List[Dict[str, object]] = [] for run_dir in self._iter_run_dirs(shallow=shallow, show_progress=False): summary = self._read_summary(run_dir) cli = self._read_cli_args(run_dir) mode = self._infer_mode(summary, cli) if mode != self.classification_mode: continue run_id = self._read_run_id(run_dir, summary) grid_params = self._grid_params_from_cli(cli, summary) rows.append( { "run_id": run_id, "run_dir": str(run_dir), "classification_mode": mode, "fusion_corrections": int(counts.get(run_id, 0)), **grid_params, } ) return pd.DataFrame(rows) @staticmethod def _aggregate_primary_metrics(df: pd.DataFrame) -> pd.DataFrame: if df.empty: return pd.DataFrame() required = ["run_id", "run_dir", "classification_mode"] if not all(col in df.columns for col in required): return pd.DataFrame() metric_cols = [ c for c in df.columns if c.startswith(("best_", "holdout_best_")) and not c.endswith(("_mean", "_sd")) ] if not metric_cols: return pd.DataFrame() grouped = df.groupby(required) agg = grouped[metric_cols].agg(["mean", "std"]) agg.columns = [ f"{col}_{stat}".replace("std", "sd") for col, stat in agg.columns ] return agg.reset_index() def _fold_sample_and_opportunity_counts(self, df: pd.DataFrame) -> pd.DataFrame: if df.empty: return pd.DataFrame() cols = ["run_id", "run_dir", "fold"] rows = [] for run_id, run_dir, fold in df[cols].drop_duplicates().itertuples(index=False): run_path = Path(run_dir) y_true = self._load_y_true(run_path, int(fold)) n_samples = int(len(y_true)) if y_true is not None else np.nan both_wrong = self._count_both_wrong(run_path, int(fold), y_true) rows.append( { "run_id": run_id, "run_dir": str(run_path), "fold": int(fold), "n_samples": n_samples, "both_wrong": both_wrong, } ) return pd.DataFrame(rows) @staticmethod def _count_both_wrong( run_dir: Path, fold: int, y_true: Optional[np.ndarray] ) -> Optional[int]: if y_true is None: return np.nan img_path = run_dir / f"fold{fold}_probs_img.npy" md_path = run_dir / f"fold{fold}_probs_md.npy" if not img_path.exists() or not md_path.exists(): return np.nan try: p_img = np.load(img_path) p_md = np.load(md_path) except Exception: return np.nan if p_img.ndim != 2 or p_md.ndim != 2: return np.nan if len(p_img) != len(y_true) or len(p_md) != len(y_true): return np.nan img_pred = p_img.argmax(axis=1) md_pred = p_md.argmax(axis=1) both_wrong = (img_pred != y_true) & (md_pred != y_true) return int(both_wrong.sum()) @staticmethod def _add_fusion_gain_columns(df: pd.DataFrame) -> pd.DataFrame: out = df.copy() for prefix in ("best", "holdout_best"): acc_cols = [ f"{prefix}_acc_fused", f"{prefix}_acc_image", f"{prefix}_acc_metadata", ] auc_cols = [ f"{prefix}_auc_fused", f"{prefix}_auc_image", f"{prefix}_auc_metadata", ] if all(c in out.columns for c in acc_cols): max_acc = out[[acc_cols[1], acc_cols[2]]].max(axis=1) out[f"{prefix}_acc_fusion_gain"] = out[acc_cols[0]] - max_acc if all(c in out.columns for c in auc_cols): max_auc = out[[auc_cols[1], auc_cols[2]]].max(axis=1) out[f"{prefix}_auc_fusion_gain"] = out[auc_cols[0]] - max_auc return out def _fusion_errors_counts(self, merged: pd.DataFrame) -> pd.DataFrame: if self.fusion_errors.empty: return pd.DataFrame(columns=["run_id", "fold", "fusion_errors"]) counts = ( self.fusion_errors.groupby(["run_id", "fold"], as_index=False) .size() .rename(columns={"size": "fusion_errors"}) ) return counts def _build_warmup_map(self, df: pd.DataFrame) -> pd.DataFrame: if df.empty or "run_id" not in df.columns or "run_dir" not in df.columns: return pd.DataFrame(columns=["run_id", "warmup_end"]) rows = [] for run_id, run_dir in ( df[["run_id", "run_dir"]].drop_duplicates().itertuples(index=False) ): cli = self._read_cli_args(Path(run_dir)) warmup_end = self._warmup_end_epoch(cli) rows.append({"run_id": run_id, "warmup_end": warmup_end}) return pd.DataFrame(rows) @staticmethod def _warmup_end_epoch(cli: Optional[Dict[str, object]]) -> int: if not cli: return 0 tower = cli.get("warmup_tower_epochs") fused = cli.get("warmup_fused_epochs") try: tower_val = int(tower) if tower is not None else 0 except Exception: tower_val = 0 try: fused_val = int(fused) if fused is not None else 0 except Exception: fused_val = 0 return max(0, tower_val + fused_val) def _grid_params_from_cli( self, cli: Optional[Dict[str, object]], summary: Optional[Dict[str, object]] ) -> Dict[str, object]: params: Dict[str, object] = { "eval_mode": None, "crop_variant": None, "crop_normalize": None, "crop_weights": None, "crop_tta": None, "loss_mode": None, "thaw_mode": None, "se_mode": None, "se_bridge_pre_norm": None, "se_tower_pre_norm": None, } eval_mode = None for payload in (cli, summary): if payload and isinstance(payload.get("eval_mode"), str): eval_mode = payload["eval_mode"] break params["eval_mode"] = eval_mode if cli: crop_weights = cli.get("img_crop_weights") crop_normalize = cli.get("img_crop_normalize") crop_tta = cli.get("img_crop_tta") params["crop_weights"] = crop_weights params["crop_normalize"] = crop_normalize params["crop_tta"] = crop_tta params["crop_variant"] = self._infer_crop_variant(crop_weights) params["loss_mode"] = self._infer_loss_mode(cli) params["thaw_mode"] = self._infer_thaw_mode(cli) params["se_mode"] = self._infer_se_mode(cli) params["se_bridge_pre_norm"] = cli.get("se_pre_norm") params["se_tower_pre_norm"] = cli.get("se_pre_norm_tower") return params @staticmethod def _infer_crop_variant(crop_weights: object) -> Optional[str]: if not crop_weights: return None text = str(crop_weights) for key in ("norm_imagenet", "normalize_none", "norm_per_image"): if key in text: return key return None @staticmethod def _infer_loss_mode(cli: Dict[str, object]) -> Optional[str]: if cli.get("balanced_sampler"): return "balanced" gamma = cli.get("focal_gamma") try: if gamma is not None and float(gamma) > 0: return "focal" except Exception: pass return "none" @staticmethod def _infer_thaw_mode(cli: Dict[str, object]) -> Optional[str]: return "gradual" if cli.get("gradual_thaw") else "none" @staticmethod def _infer_se_mode(cli: Dict[str, object]) -> Optional[str]: if not cli.get("use_se"): return "none" se_where = cli.get("se_where") if isinstance(se_where, str) and se_where.strip(): return se_where.strip() return "bridge" @staticmethod def _read_json(path: Path) -> Optional[Dict[str, object]]: if not path.exists(): return None try: data = json.loads(path.read_text()) except Exception: return None if not isinstance(data, dict): return None return data @staticmethod def _read_epoch_log(run_dir: Path, fold: int) -> Optional[pd.DataFrame]: path = run_dir / f"fold{fold}_epoch_log.csv" if not path.exists(): return None try: return pd.read_csv(path) except Exception: return None def _read_summary(self, run_dir: Path) -> Optional[Dict[str, object]]: return self._read_json(run_dir / "summary.json") def _read_cli_args(self, run_dir: Path) -> Optional[Dict[str, object]]: return self._read_json(run_dir / "cli_args.json") @staticmethod def _read_run_id(run_dir: Path, summary: Optional[Dict[str, object]]) -> str: if summary: rid = summary.get("run_id") if isinstance(rid, str) and rid: return rid return run_dir.name @staticmethod def _infer_mode( summary: Optional[Dict[str, object]], cli: Optional[Dict[str, object]] ) -> Optional[str]: for payload in (summary, cli): if not payload: continue eval_mode = payload.get("eval_mode") if isinstance(eval_mode, str): mode = eval_mode.strip().lower() if mode == "binary": return "binary" if mode in {"multiclass", "multi", "multi-class"}: return "multiclass" num_classes = payload.get("num_classes") if isinstance(num_classes, (int, float)): return "binary" if int(num_classes) <= 2 else "multiclass" class_names = payload.get("class_names") if isinstance(class_names, list) and class_names: return "binary" if len(class_names) <= 2 else "multiclass" return None @staticmethod def _fold_epoch_hint( summary: Optional[Dict[str, object]], fold: int ) -> Optional[int]: if not summary: return None fold_metrics = summary.get("fold_metrics") or [] for entry in fold_metrics: if not isinstance(entry, dict): continue if entry.get("fold") == fold: stats = ( entry.get("stats") if isinstance(entry.get("stats"), dict) else {} ) epoch = stats.get("epoch") or entry.get("best_epoch") if isinstance(epoch, (int, float)): return int(epoch) return None @staticmethod def _extract_best_epochs( log_df: Optional[pd.DataFrame], ) -> tuple[Optional[int], Optional[int]]: if log_df is None or log_df.empty: return None, None best_epoch = None holdout_best_epoch = None if "best_epoch" in log_df.columns: try: best_epoch = int(log_df["best_epoch"].iloc[-1]) except Exception: best_epoch = None if "holdout_best_epoch" in log_df.columns: try: holdout_best_epoch = int(log_df["holdout_best_epoch"].iloc[-1]) except Exception: holdout_best_epoch = None return best_epoch, holdout_best_epoch @staticmethod def _row_for_epoch( log_df: Optional[pd.DataFrame], epoch: Optional[int] ) -> Optional[pd.Series]: if log_df is None or epoch is None: return None if "epoch" not in log_df.columns: return None rows = log_df[log_df["epoch"] == epoch] if rows.empty: return None return rows.iloc[-1] @staticmethod def _metric_from_row(row: Optional[pd.Series], key: str) -> Optional[float]: if row is None or key not in row: return None try: val = float(row[key]) except Exception: return None if np.isnan(val): return None return val @staticmethod def _available_folds( run_dir: Path, summary: Optional[Dict[str, object]] ) -> List[int]: folds: List[int] = [] if summary: for entry in summary.get("fold_metrics") or []: if not isinstance(entry, dict): continue fold_idx = entry.get("fold") if isinstance(fold_idx, int): folds.append(fold_idx) if not folds: pattern = re.compile(r"fold(\d+)_y_true\.npy$") for path in run_dir.glob("fold*_y_true.npy"): match = pattern.match(path.name) if match: folds.append(int(match.group(1))) return sorted(set(folds)) @staticmethod def _load_y_true(run_dir: Path, fold: int) -> Optional[np.ndarray]: path = run_dir / f"fold{fold}_y_true.npy" if not path.exists(): return None try: return np.load(path) except Exception: return None @staticmethod def _collect_epoch_prob_paths( run_dir: Path, fold: int ) -> Dict[int, Dict[str, Path]]: pattern = re.compile(rf"fold{fold}_epoch(\d+)_probs_(\w+)\.npy$") epoch_paths: Dict[int, Dict[str, Path]] = {} for path in run_dir.glob(f"fold{fold}_epoch*_probs_*.npy"): match = pattern.match(path.name) if not match: continue epoch = int(match.group(1)) head = match.group(2) epoch_paths.setdefault(epoch, {})[head] = path return epoch_paths @staticmethod def _collect_base_prob_paths(run_dir: Path, fold: int) -> Dict[str, Path]: paths: Dict[str, Path] = {} for head in ("fused", "img", "md"): candidate = run_dir / f"fold{fold}_probs_{head}.npy" if candidate.exists(): paths[head] = candidate return paths @staticmethod def _load_probs_array(path: Path) -> Optional[np.ndarray]: try: return np.load(path) except Exception: return None def _load_auc_for_epoch( self, run_dir: Path, fold: int, epoch: int, head: str, holdout: bool ) -> Optional[float]: if epoch is None: return None tag = "holdout_" if holdout else "" # Prefer per-epoch ROC curves (same evaluation used for epoch_log ACC). epoch_dir = run_dir / f"fold{fold}_roc_curves" if epoch_dir.exists(): path = epoch_dir / f"epoch{epoch}_{tag}{head}.json" else: path = None # Fallback to best/holdout_best exports if epoch curves missing. if path is None or not path.exists(): folder = ( "fold{}_roc_curves_holdout_best".format(fold) if holdout else "fold{}_roc_curves_best".format(fold) ) target_dir = run_dir / folder path = target_dir / f"epoch{epoch}_{tag}{head}.json" if not path.exists(): return None return self._extract_auc_from_json(path) @staticmethod def _extract_auc_from_json(path: Path) -> Optional[float]: try: payload = json.loads(path.read_text()) except Exception: return None if not isinstance(payload, dict): return None per_class = payload.get("per_class") if isinstance(per_class, dict): def _to_float(val): try: f = float(val) except Exception: return None if np.isnan(f): return None return f # Binary fix: class 0 stored with class-1 labels against class-0 scores. if "0" in per_class and "1" in per_class: v0 = _to_float(per_class.get("0", {}).get("auc")) v1 = _to_float(per_class.get("1", {}).get("auc")) if v0 is not None and v1 is None: return float(1.0 - v0) if v1 is not None and v0 is None: return float(1.0 - v1) if v0 is not None and v1 is not None: return float(np.mean([v0, v1])) vals = [] for entry in per_class.values(): if not isinstance(entry, dict): continue auc_val = entry.get("auc") val = _to_float(auc_val) if val is not None: vals.append(val) if vals: return float(np.mean(vals)) macro_auc = payload.get("macro_auc") try: if macro_auc is not None and not np.isnan(float(macro_auc)): return float(macro_auc) except Exception: pass return None @staticmethod def _plot_correlation_bars( df: pd.DataFrame, path: Path, method: str = "pearson", x_metric: str = "fusion_corrections", metric_type: str = "acc", title: Optional[str] = None, ) -> None: if df.empty: return height = max(4.0, 0.28 * len(df)) fig, ax = plt.subplots(figsize=(8.5, height)) ax.barh(df["metric"], df["corr"], color="steelblue") ax.axvline(0.0, color="black", lw=1) label = "Spearman ρ" if method == "spearman" else "Pearson r" ax.set_xlabel(label) ax.set_title(title or f"{x_metric} vs {metric_type} metrics ({method})") fig.tight_layout() fig.savefig(path, dpi=170) plt.close(fig) @staticmethod def _prepare_probs(arr: np.ndarray) -> Optional[np.ndarray]: if arr is None: return None probs = np.asarray(arr, dtype=float) if probs.ndim == 1: probs = np.stack([1.0 - probs, probs], axis=1) if probs.ndim != 2: return None return probs @staticmethod def _has_all_heads(arrays: Dict[str, Optional[np.ndarray]]) -> bool: needed = ("fused", "img", "md") return all(arrays.get(head) is not None for head in needed) def _fusion_corrections_for_probs( self, y_true: np.ndarray, arrays: Dict[str, np.ndarray], run_id: str, fold: int, epoch: int, ) -> List[Dict[str, object]]: fused = self._prepare_probs(arrays.get("fused")) img = self._prepare_probs(arrays.get("img")) md = self._prepare_probs(arrays.get("md")) if fused is None or img is None or md is None: return [] if not (len(fused) == len(img) == len(md) == len(y_true)): return [] fused_pred = fused.argmax(axis=1) img_pred = img.argmax(axis=1) md_pred = md.argmax(axis=1) fused_conf = np.take_along_axis(fused, fused_pred[:, None], axis=1).squeeze(1) img_conf = np.take_along_axis(img, img_pred[:, None], axis=1).squeeze(1) md_conf = np.take_along_axis(md, md_pred[:, None], axis=1).squeeze(1) mask = (fused_pred == y_true) & (img_pred != y_true) & (md_pred != y_true) indices = np.nonzero(mask)[0] events: List[Dict[str, object]] = [] for idx in indices: events.append( { "run_id": run_id, "fold": fold, "epoch": epoch, "index": int(idx), "y_true": int(y_true[idx]), "pred_fused": int(fused_pred[idx]), "pred_img": int(img_pred[idx]), "pred_md": int(md_pred[idx]), "conf_fused": float(fused_conf[idx]), "conf_img": float(img_conf[idx]), "conf_md": float(md_conf[idx]), } ) return events def _fusion_errors_for_probs( self, y_true: np.ndarray, arrays: Dict[str, np.ndarray], run_id: str, fold: int, epoch: int, ) -> List[Dict[str, object]]: fused = self._prepare_probs(arrays.get("fused")) img = self._prepare_probs(arrays.get("img")) md = self._prepare_probs(arrays.get("md")) if fused is None or img is None or md is None: return [] if not (len(fused) == len(img) == len(md) == len(y_true)): return [] fused_pred = fused.argmax(axis=1) img_pred = img.argmax(axis=1) md_pred = md.argmax(axis=1) fused_conf = np.take_along_axis(fused, fused_pred[:, None], axis=1).squeeze(1) img_conf = np.take_along_axis(img, img_pred[:, None], axis=1).squeeze(1) md_conf = np.take_along_axis(md, md_pred[:, None], axis=1).squeeze(1) mask = (fused_pred != y_true) & (img_pred == y_true) & (md_pred == y_true) indices = np.nonzero(mask)[0] events: List[Dict[str, object]] = [] for idx in indices: events.append( { "run_id": run_id, "fold": fold, "epoch": epoch, "index": int(idx), "y_true": int(y_true[idx]), "pred_fused": int(fused_pred[idx]), "pred_img": int(img_pred[idx]), "pred_md": int(md_pred[idx]), "conf_fused": float(fused_conf[idx]), "conf_img": float(img_conf[idx]), "conf_md": float(md_conf[idx]), } ) return events if __name__ == "__main__": analysis = derived_analysis( Path("analysis_data/grid_search"), classification_mode="binary" ) df = analysis.identify_fusion_corrections(existing=True) # analysis.write_fusion_corrections() # analysis.write_fusion_errors() print(f"Fusion correction events: {len(df)}") analysis.primary_metrics = analysis.populate_primary_metrics(existing=True) # analysis.write_primary_metrics() # corr_acc = analysis.fusion_corrections_correlation( # method="spearman", metric_type="acc" # ) # corr_auc = analysis.fusion_corrections_correlation( # method="spearman", metric_type="auc" # ) # print(corr_acc) # summary = analysis.plot_fusion_perf_summary( # corr_acc, # corr_auc, # method="spearman", # x_metric="fusion_corrections_per_opportunity", # ) analysis.primary_metrics df = analysis.param_performance_correlations(method="spearman") df.columns # analysis.plot_param_perf_corr_panels(df) out = analysis.se_mode_effects_summary(metric="auc") out2 = analysis.se_mode_effects_summary(metric="acc") out3 = analysis.se_mode_effects_summary(metric="auc", head="fused", prefer_holdout=True, top_n=25) out4 = analysis.se_mode_effects_summary(metric="acc", head="fused", prefer_holdout=True, top_n=25) print(out["summary"]) print(out2["summary"]) print(out3["summary"]) print(out4["summary"]) print(out["pairwise"]) print(out3["pairwise"]) # analysis.fusion_corrections # analysis.plot_fusion_corrections_errors() # analysis.plot_conf_delta_boxplot()