v4 update

v4/classes/metrics.py

@@ -0,0 +1,175 @@
+"""metrics — loss, scoring, calibration, and threshold/bias tuning."""
+from __future__ import annotations
+
+from typing import Optional
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from sklearn.metrics import (
+    cohen_kappa_score,
+    f1_score,
+    matthews_corrcoef,
+    recall_score,
+    roc_auc_score,
+    roc_curve,
+)
+
+
+# ---------------------------------------------------------------------------
+# Loss
+# ---------------------------------------------------------------------------
+
+def focal_loss(
+    logits:    torch.Tensor,
+    targets:   torch.Tensor,
+    gamma:     float = 0.0,
+    weight:    Optional[torch.Tensor] = None,
+    reduction: str = "mean",
+) -> torch.Tensor:
+    """Focal loss; reduces to cross-entropy when gamma=0."""
+    if gamma <= 0:
+        return F.cross_entropy(logits, targets, weight=weight, reduction=reduction)
+    log_probs = F.log_softmax(logits, dim=1)
+    probs     = log_probs.exp()
+    targets   = targets.long().view(-1, 1)
+    logpt     = log_probs.gather(1, targets)
+    pt        = probs.gather(1, targets)
+    loss      = -(((1.0 - pt).clamp_min(0.0) ** gamma) * logpt)
+    if weight is not None:
+        loss = loss * weight.gather(0, targets.view(-1)).view(-1, 1)
+    loss = loss.view(-1)
+    if reduction == "sum":  return loss.sum()
+    if reduction == "mean": return loss.mean()
+    return loss
+
+
+# ---------------------------------------------------------------------------
+# Basic array scoring
+# ---------------------------------------------------------------------------
+
+def score_arrays(y_true: np.ndarray, probs: np.ndarray, num_classes: int):
+    """Return (acc, auc, n)."""
+    if y_true.size == 0:
+        return float("nan"), float("nan"), 0
+    acc = float((probs.argmax(1) == y_true).mean())
+    try:
+        auc = (
+            float(roc_auc_score(y_true, probs[:, 1]))
+            if num_classes == 2
+            else float(roc_auc_score(y_true, probs, multi_class="ovr", average="macro"))
+        )
+    except Exception:
+        auc = float("nan")
+    return acc, auc, int(len(y_true))
+
+
+# ---------------------------------------------------------------------------
+# Calibration
+# ---------------------------------------------------------------------------
+
+def compute_ece(y_true: np.ndarray, probs: np.ndarray, n_bins: int = 10) -> float:
+    """Expected Calibration Error: weighted mean |confidence − accuracy| per bin."""
+    if y_true.size == 0:
+        return float("nan")
+    confidences = probs.max(axis=1)
+    predictions = probs.argmax(axis=1)
+    bin_edges   = np.linspace(0.0, 1.0, n_bins + 1)
+    ece = 0.0
+    n   = len(y_true)
+    for i, (lo, hi) in enumerate(zip(bin_edges[:-1], bin_edges[1:])):
+        mask = (confidences >= lo) & (
+            confidences <= hi if i == n_bins - 1 else confidences < hi
+        )
+        if not mask.any():
+            continue
+        ece += float(mask.sum()) / n * abs(
+            float(confidences[mask].mean()) - float((predictions[mask] == y_true[mask]).mean())
+        )
+    return float(ece)
+
+
+def compute_extended_metrics(
+    y_true:          np.ndarray,
+    probs:           np.ndarray,
+    num_classes:     int,
+    n_bins:          int = 10,
+    preds_override:  Optional[np.ndarray] = None,
+) -> dict:
+    nan = float("nan")
+    if y_true.size == 0:
+        return dict(
+            kappa=nan, mcc=nan, macro_f1=nan,
+            per_class_recall=np.full(num_classes, nan), ece=nan,
+        )
+    preds = preds_override if preds_override is not None else probs.argmax(axis=1)
+    try:    kappa    = float(cohen_kappa_score(y_true, preds))
+    except: kappa    = nan
+    try:    mcc      = float(matthews_corrcoef(y_true, preds))
+    except: mcc      = nan
+    try:    macro_f1 = float(f1_score(y_true, preds, average="macro", zero_division=0))
+    except: macro_f1 = nan
+    try:
+        pcr = recall_score(
+            y_true, preds, average=None,
+            labels=list(range(num_classes)), zero_division=0,
+        ).astype(float)
+    except:
+        pcr = np.full(num_classes, nan)
+    return dict(
+        kappa=kappa, mcc=mcc, macro_f1=macro_f1,
+        per_class_recall=pcr, ece=compute_ece(y_true, probs, n_bins=n_bins),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Threshold / bias tuning
+# ---------------------------------------------------------------------------
+
+def tune_binary_threshold(y_true: np.ndarray, p1: np.ndarray) -> float:
+    """Pick threshold via Youden's J (sensitivity + specificity − 1).
+
+    Class-distribution independent; falls back to 0.5 if fewer than two
+    classes are present in y_true.
+    """
+    if y_true.size == 0 or len(np.unique(y_true)) < 2:
+        return 0.5
+    fpr, tpr, thresholds = roc_curve(y_true, p1)
+    return float(thresholds[np.argmax(tpr + (1.0 - fpr) - 1.0)])
+
+
+def multiclass_acc_with_bias(
+    y_true: np.ndarray, probs: np.ndarray, bias: np.ndarray
+) -> float:
+    """Balanced accuracy (mean per-class recall) after applying log-space bias."""
+    if y_true.size == 0:
+        return float("nan")
+    logits  = np.log(np.clip(probs, 1e-8, 1.0)) + bias.reshape(1, -1)
+    preds   = np.argmax(logits, axis=1)
+    classes = np.unique(y_true)
+    return float(np.mean([(preds[y_true == c] == c).mean() for c in classes]))
+
+
+def tune_multiclass_bias(
+    y_true: np.ndarray, probs: np.ndarray, *, iters: int = 2
+) -> np.ndarray:
+    """Grid-search per-class log-space bias to maximise balanced accuracy."""
+    if y_true.size == 0 or probs.size == 0:
+        return np.zeros((0,), dtype=float)
+    c    = probs.shape[1]
+    bias = np.zeros((c,), dtype=float)
+    grid = np.linspace(-1.0, 1.0, 41)
+    for _ in range(iters):
+        for k in range(c):
+            best_v   = bias[k]
+            best_acc = multiclass_acc_with_bias(y_true, probs, bias)
+            old      = bias[k]
+            for v in grid:
+                bias[k] = float(v)
+                acc = multiclass_acc_with_bias(y_true, probs, bias)
+                if acc > best_acc or (acc == best_acc and abs(v) < abs(best_v)):
+                    best_acc, best_v = acc, float(v)
+            bias[k] = best_v
+            if np.isnan(best_acc):
+                bias[k] = old
+    return bias