Add new regression and ensemble experiment configurations for V2-M and OrthoBridge
- Introduced multiple regression experiment configurations targeting vf_md, including: - cd_solo_reg_set.json: CD tower only regression setup. - img_solo_reg_set.json: Image tower only regression setup. - reg_head_epoch_sweep.json: Baseline regression sweeps at different epochs (50, 75, 100). - reg_head_set.json: Various regression setups including baseline and OrthoBridge configurations. - single_eye_reg.json: Single-eye regression setup for worst-eye aggregation analysis. - Added ensemble configurations for OrthoBridge with different inner bridges: - ortho_alts_ensemble.json: Ensemble tests with ConcatBridge, PairwiseAdditiveBridge, and GatedAdditiveBridge. - ortho_alts_tritower.json: Tritower tests with the same inner bridges. - Created V2-M specific configurations: - baseline_reg_nt50.json: Regression baseline with V2-M backbone. - geom_vec_gt.json and geom_vec_unet.json: Geometry vector injection experiments with V2-M. - single_l1_bridges.json: Single-eye ensemble experiments with various bridge types. - tritower_geom_gt.json: Tritower setup with GT contour-rasterized masks. - Promoted existing experiments to higher repetitions for robustness.
This commit is contained in:
rpotter6298
@@ -0,0 +1,221 @@
|
||||
"""F2 — Backbone selection panel.
|
||||
|
||||
Box plot in the style of v3/figures/phase2_analysis.png (black-bordered boxes,
|
||||
red median lines, baseline median reference). Three left-to-right sections:
|
||||
|
||||
Block 1 (blue) — Basic backbones (img-only, single-eye, ImageNet pretraining):
|
||||
VGG16, MobileNetV2, DenseNet121, InceptionV3, ResNet50
|
||||
Sourced from v3 phase 1 / phase 2 fold AUCs. Will be refined with v4
|
||||
10x5 runs later; means should not move much.
|
||||
|
||||
Block 2 (blue) — ResNet50 preprocessing/CV variations:
|
||||
leaky CV, GT crop, U-Net crop (all 2.5x scale; 1.1x dropped from labels)
|
||||
Sourced from v3 phase 2 'classic_test_auc' (single-mode image-only).
|
||||
|
||||
Block 3 (orange) — Baseline reference:
|
||||
"Baseline (fine-tuned ResNet50)" — what we previously called refugelike.
|
||||
Sourced from v3 phase 2 imageonly_refugelike_proper.
|
||||
|
||||
Each non-baseline box is labelled with a Wilcoxon two-sided p-value comparing
|
||||
its fold AUCs to the baseline.
|
||||
|
||||
Re-run anytime:
|
||||
python -m v4.figures.F2_papila_replication_and_single_mode
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from scipy.stats import wilcoxon
|
||||
|
||||
from v4.figures.util.loaders import REPO_ROOT
|
||||
|
||||
OUT = Path(__file__).parent / "output" / "F2_backbones.png"
|
||||
|
||||
# ── Colors / styling (mirrors v3 phase2_analysis) ────────────────────────────
|
||||
C_VAR = "#4c72b0" # blue — non-baseline boxes (basic backbones + variants)
|
||||
C_BASE = "#dd8452" # orange — baseline reference box
|
||||
C_MEDIAN = "#c44e52" # red — median line inside boxes
|
||||
ALPHA = 0.82
|
||||
|
||||
V3_PHASE1_DIR = REPO_ROOT / "v3" / "results" / "phase1"
|
||||
V3_PHASE2_DIR = REPO_ROOT / "v3" / "results" / "phase2"
|
||||
|
||||
|
||||
def _wilcoxon_p(a: np.ndarray, b: np.ndarray) -> float:
|
||||
diffs = a - b
|
||||
if len(diffs) < 5 or np.all(diffs == 0):
|
||||
return float("nan")
|
||||
try:
|
||||
return float(wilcoxon(diffs, alternative="two-sided").pvalue)
|
||||
except Exception:
|
||||
return float("nan")
|
||||
|
||||
|
||||
def _load_phase1_fold_aucs(subdir: str) -> np.ndarray:
|
||||
fp = V3_PHASE1_DIR / subdir / "fold_metrics.csv"
|
||||
if not fp.exists():
|
||||
return np.array([])
|
||||
df = pd.read_csv(fp)
|
||||
return df["auc"].dropna().astype(float).values
|
||||
|
||||
|
||||
def _load_phase2_classic_aucs(run_name: str) -> np.ndarray:
|
||||
"""Collect classic_test_auc across all rep×fold for a phase 2 run folder."""
|
||||
root = V3_PHASE2_DIR / run_name
|
||||
if not root.exists():
|
||||
return np.array([])
|
||||
out: list[float] = []
|
||||
for rep in sorted(root.glob("rep*")):
|
||||
fp = rep / "binary" / "single" / "fold_results.csv"
|
||||
if not fp.exists(): continue
|
||||
df = pd.read_csv(fp)
|
||||
if "classic_test_auc" not in df.columns: continue
|
||||
out.extend(df["classic_test_auc"].dropna().astype(float).tolist())
|
||||
return np.array(out)
|
||||
|
||||
|
||||
# ── Per-section data definitions ─────────────────────────────────────────────
|
||||
# Each entry: (label, loader_fn, *args)
|
||||
|
||||
BASIC_BACKBONES = [
|
||||
("VGG16", _load_phase1_fold_aucs, "cnn_vgg16"),
|
||||
("MobileNetV2", _load_phase1_fold_aucs, "cnn_mobilenet_v2"),
|
||||
("DenseNet121", _load_phase1_fold_aucs, "cnn_densenet121"),
|
||||
("InceptionV3", _load_phase1_fold_aucs, "cnn_inception_v3"),
|
||||
# Use phase 2 ResNet50 (50 fold AUCs) for tighter statistics on the
|
||||
# backbone that we sweep variations of in block 2.
|
||||
("ResNet50", _load_phase2_classic_aucs, "imageonly_resnet50_proper"),
|
||||
]
|
||||
|
||||
RESNET_VARIATIONS = [
|
||||
("leaky CV", _load_phase2_classic_aucs, "imageonly_resnet50_leaky"),
|
||||
("GT crop", _load_phase2_classic_aucs, "imageonly_resnet50_gtcrop_2.5"),
|
||||
("U-Net crop", _load_phase2_classic_aucs, "imageonly_resnet50_unetcrop_2.5"),
|
||||
]
|
||||
|
||||
BASELINE_LABEL = "baseline\n(fine-tuned ResNet50)"
|
||||
BASELINE_DATA = (_load_phase2_classic_aucs, "imageonly_refugelike_proper")
|
||||
|
||||
|
||||
def render() -> None:
|
||||
# Load everything
|
||||
block1 = [(lbl, fn(arg)) for lbl, fn, arg in BASIC_BACKBONES]
|
||||
block2 = [(lbl, fn(arg)) for lbl, fn, arg in RESNET_VARIATIONS]
|
||||
base_fn, base_arg = BASELINE_DATA
|
||||
base_aucs = base_fn(base_arg)
|
||||
|
||||
print("Block 1 — Basic backbones:")
|
||||
for lbl, a in block1:
|
||||
print(f" {lbl:<14s} n={len(a):>3d} mean={a.mean():.3f}±{a.std():.3f}" if len(a) else f" {lbl:<14s} no data")
|
||||
print("Block 2 — ResNet50 variations:")
|
||||
for lbl, a in block2:
|
||||
print(f" {lbl:<14s} n={len(a):>3d} mean={a.mean():.3f}±{a.std():.3f}" if len(a) else f" {lbl:<14s} no data")
|
||||
print(f"Block 3 — Baseline: n={len(base_aucs)} "
|
||||
f"mean={base_aucs.mean():.3f}±{base_aucs.std():.3f}" if len(base_aucs) else "Block 3 — no baseline data")
|
||||
|
||||
# Lay out positions
|
||||
gap = 0.7
|
||||
pos: list[float] = []
|
||||
p = 0.0
|
||||
for _ in block1:
|
||||
pos.append(p); p += 1.0
|
||||
section1_right = p - 1.0
|
||||
p += gap
|
||||
section2_left = p
|
||||
for _ in block2:
|
||||
pos.append(p); p += 1.0
|
||||
section2_right = p - 1.0
|
||||
p += gap
|
||||
section3_left = p
|
||||
pos.append(p)
|
||||
section3_right = p
|
||||
total_w = p + 0.6
|
||||
|
||||
fig, ax = plt.subplots(figsize=(13, 5.8))
|
||||
fig.suptitle("Backbone Selection", fontsize=13, fontweight="bold")
|
||||
|
||||
box_w = 0.55
|
||||
boxprops_kw = dict(linewidth=1.2, edgecolor="black")
|
||||
medianprops = dict(color=C_MEDIAN, linewidth=2)
|
||||
whiskerprops = dict(color="black", linewidth=1.0)
|
||||
capprops = dict(color="black", linewidth=1.0)
|
||||
flierprops = dict(marker="o", markersize=3, alpha=0.55,
|
||||
markerfacecolor="#888", markeredgecolor="#444")
|
||||
|
||||
all_aucs: list[np.ndarray] = []
|
||||
all_labels: list[str] = []
|
||||
all_colors: list[str] = []
|
||||
|
||||
for lbl, a in block1 + block2:
|
||||
all_labels.append(lbl); all_aucs.append(a); all_colors.append(C_VAR)
|
||||
all_labels.append(BASELINE_LABEL); all_aucs.append(base_aucs); all_colors.append(C_BASE)
|
||||
|
||||
# Draw boxes
|
||||
for x, aucs, color in zip(pos, all_aucs, all_colors):
|
||||
if not len(aucs): continue
|
||||
bp = ax.boxplot(
|
||||
aucs, positions=[x], widths=box_w, patch_artist=True, manage_ticks=False,
|
||||
boxprops=dict(facecolor=color, alpha=ALPHA, **boxprops_kw),
|
||||
medianprops=medianprops,
|
||||
whiskerprops=whiskerprops,
|
||||
capprops=capprops,
|
||||
flierprops=flierprops,
|
||||
)
|
||||
|
||||
# Baseline median reference line spanning the variant blocks
|
||||
if len(base_aucs):
|
||||
ax.axhline(np.median(base_aucs),
|
||||
color=C_BASE, linewidth=1.2, linestyle="--", alpha=0.55,
|
||||
label="Baseline median")
|
||||
|
||||
# Dividers between sections (vertical light lines)
|
||||
div1 = (section1_right + section2_left) / 2
|
||||
div2 = (section2_right + section3_left) / 2
|
||||
for d in (div1, div2):
|
||||
ax.axvline(d, color="#aaa", linewidth=0.7, alpha=0.65, linestyle="-")
|
||||
|
||||
# Section labels just above each block
|
||||
y_band = 1.02
|
||||
section_centers = [
|
||||
((pos[0] + section1_right) / 2, "Basic backbones (img-only, single)"),
|
||||
((section2_left + section2_right) / 2, "ResNet50 variations"),
|
||||
((section3_left + section3_right) / 2, "Baseline"),
|
||||
]
|
||||
for cx, txt in section_centers:
|
||||
ax.text(cx, y_band, txt, ha="center", va="bottom",
|
||||
fontsize=10, color="#333", fontweight="bold",
|
||||
transform=ax.get_xaxis_transform())
|
||||
|
||||
# X-tick labels (with p-values vs baseline beneath each variant box)
|
||||
tick_labels = []
|
||||
for lbl, aucs, color in zip(all_labels, all_aucs, all_colors):
|
||||
if color == C_BASE or not len(aucs) or not len(base_aucs):
|
||||
tick_labels.append(lbl); continue
|
||||
n = min(len(aucs), len(base_aucs))
|
||||
p_val = _wilcoxon_p(aucs[:n], base_aucs[:n])
|
||||
ps = f"p={p_val:.3f}" if not np.isnan(p_val) else "p=n/a"
|
||||
tick_labels.append(f"{lbl}\n{ps}")
|
||||
ax.set_xticks(pos)
|
||||
ax.set_xticklabels(tick_labels, fontsize=9.5)
|
||||
|
||||
ax.set_xlim(-0.6, section3_right + 0.7)
|
||||
ax.set_ylim(0.55, 1.0)
|
||||
ax.set_ylabel("Test AUC", fontsize=11)
|
||||
ax.grid(axis="y", alpha=0.3, linestyle="--")
|
||||
ax.legend(loc="lower left", fontsize=9, framealpha=0.92)
|
||||
|
||||
fig.tight_layout()
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(OUT, dpi=180, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
print(f"saved {OUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
render()
|
||||
Reference in New Issue
Block a user