added better memory caching, multithreaded processing, cleanup scripts dir
This commit is contained in:
rpotter6298
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rank multiclass runs by mean holdout AUC (fused) across folds."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
# ---------------------------
|
||||
# Config (edit in IDE)
|
||||
# ---------------------------
|
||||
ANALYSIS_DIR = Path("analysis_data/grid_search")
|
||||
TOP_N = 20
|
||||
HEAD = "fused" # fused | image | metadata
|
||||
OUTPUT_CSV = Path("analysis_data/grid_search/plots/best_holdout_multiclass.csv")
|
||||
|
||||
|
||||
def _read_json(path: Path) -> Optional[Dict[str, Any]]:
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
data = json.loads(path.read_text())
|
||||
except Exception:
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
def _infer_mode(summary: Optional[Dict[str, Any]]) -> Optional[str]:
|
||||
if not summary:
|
||||
return None
|
||||
eval_mode = summary.get("eval_mode")
|
||||
if isinstance(eval_mode, str):
|
||||
mode = eval_mode.strip().lower()
|
||||
if mode == "binary":
|
||||
return "binary"
|
||||
if mode in {"multiclass", "multi", "multi-class"}:
|
||||
return "multiclass"
|
||||
num_classes = summary.get("num_classes")
|
||||
if isinstance(num_classes, (int, float)):
|
||||
return "binary" if int(num_classes) <= 2 else "multiclass"
|
||||
class_names = summary.get("class_names")
|
||||
if isinstance(class_names, list) and class_names:
|
||||
return "binary" if len(class_names) <= 2 else "multiclass"
|
||||
return None
|
||||
|
||||
|
||||
def _simple_fields(summary: Dict[str, Any]) -> Dict[str, Any]:
|
||||
keep: Dict[str, Any] = {}
|
||||
for key, val in summary.items():
|
||||
if key == "fold_metrics":
|
||||
continue
|
||||
if isinstance(val, (str, int, float, bool)) or val is None:
|
||||
keep[key] = val
|
||||
return keep
|
||||
|
||||
|
||||
def _collect_fold_values(summary: Dict[str, Any], metric_key: str) -> List[float]:
|
||||
values: List[float] = []
|
||||
for entry in summary.get("fold_metrics") or []:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
stats = entry.get("stats") if isinstance(entry.get("stats"), dict) else {}
|
||||
val = stats.get(metric_key)
|
||||
if isinstance(val, (int, float)):
|
||||
values.append(float(val))
|
||||
return values
|
||||
|
||||
|
||||
def main() -> None:
|
||||
metric_key = f"holdout_auc_{HEAD}"
|
||||
rows: List[Dict[str, Any]] = []
|
||||
|
||||
for run_dir in sorted(ANALYSIS_DIR.iterdir()):
|
||||
if not run_dir.is_dir():
|
||||
continue
|
||||
summary = _read_json(run_dir / "summary.json")
|
||||
mode = _infer_mode(summary)
|
||||
if mode != "multiclass":
|
||||
continue
|
||||
|
||||
values = _collect_fold_values(summary, metric_key)
|
||||
if not values:
|
||||
continue
|
||||
|
||||
mean_val = float(np.mean(values))
|
||||
std_val = float(np.std(values, ddof=1)) if len(values) > 1 else float("nan")
|
||||
|
||||
row = {
|
||||
"run_id": summary.get("run_id", run_dir.name),
|
||||
"run_dir": str(run_dir),
|
||||
"metric": metric_key,
|
||||
"mean": mean_val,
|
||||
"std": std_val,
|
||||
"n_folds": len(values),
|
||||
**_simple_fields(summary),
|
||||
}
|
||||
rows.append(row)
|
||||
|
||||
if not rows:
|
||||
raise SystemExit("No multiclass runs with holdout AUC found.")
|
||||
|
||||
df = pd.DataFrame(rows).sort_values(by="mean", ascending=False)
|
||||
top_df = df.head(TOP_N) if TOP_N else df
|
||||
|
||||
OUTPUT_CSV.parent.mkdir(parents=True, exist_ok=True)
|
||||
df.to_csv(OUTPUT_CSV, index=False)
|
||||
|
||||
print(top_df.to_string(index=False, float_format=lambda x: f"{x:.4f}"))
|
||||
print(f"\nSaved full ranking to: {OUTPUT_CSV}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,376 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare cached crop bounds/features vs GT-derived crops from the manifest."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from PIL import Image
|
||||
import torch
|
||||
from torchvision import transforms
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from classes.hypertower import ManifestImageCropper, UNetImageCropper
|
||||
from classes.refuge_segmentation import UNet as RefugeUNet
|
||||
|
||||
|
||||
# ---------------------------
|
||||
# Config (edit in IDE)
|
||||
# ---------------------------
|
||||
CACHE_DIR = Path("analysis_data/hypertower_crops")
|
||||
MANIFEST_PATH = Path("manifest.csv")
|
||||
IMAGE_DIR = Path("Papila/FundusImages")
|
||||
SCALE = 2.5
|
||||
MAX_SAMPLES = 200 # set None to scan all
|
||||
TOL_BOUNDS = 1.0 # pixels
|
||||
TOL_FEATURES = 1e-3
|
||||
UNET_VARIANTS = [
|
||||
("norm_imagenet", Path("models/unet_segmenter/norm_imagenet/best.pt"), "imagenet"),
|
||||
("normalize_none", Path("models/unet_segmenter/normalize_none/best.pt"), "none"),
|
||||
("norm_per_image", Path("models/unet_segmenter/norm_per_image/best.pt"), "per_image"),
|
||||
]
|
||||
REFUGE_SEG_WEIGHTS = Path("models/refuge/segmentation/refuge_segmentation_best.pt")
|
||||
|
||||
|
||||
def _load_cache(path: Path) -> Optional[Dict[str, np.ndarray]]:
|
||||
try:
|
||||
data = np.load(path, allow_pickle=False)
|
||||
except Exception:
|
||||
return None
|
||||
return {k: data[k] for k in data.files}
|
||||
|
||||
|
||||
def _parse_stem(path: Path) -> str:
|
||||
# expects RET###OS_s250.npz -> RET###OS
|
||||
stem = path.stem
|
||||
if "_s" in stem:
|
||||
stem = stem.split("_s")[0]
|
||||
return stem
|
||||
|
||||
|
||||
def _image_path_from_stem(stem: str) -> Optional[Path]:
|
||||
cand = IMAGE_DIR / f"{stem}.jpg"
|
||||
if cand.exists():
|
||||
return cand
|
||||
cand = IMAGE_DIR / f"{stem}.png"
|
||||
if cand.exists():
|
||||
return cand
|
||||
return None
|
||||
|
||||
|
||||
def _gt_info(
|
||||
cropper: ManifestImageCropper, image_path: Path
|
||||
) -> Optional[Dict[str, float]]:
|
||||
try:
|
||||
image = Image.open(image_path).convert("RGB")
|
||||
except Exception:
|
||||
return None
|
||||
info = cropper._compute_crop_info(image, image_path)
|
||||
return info
|
||||
|
||||
|
||||
def _unet_info(
|
||||
cropper: UNetImageCropper, image_path: Path
|
||||
) -> Optional[Dict[str, float]]:
|
||||
try:
|
||||
image = Image.open(image_path).convert("RGB")
|
||||
except Exception:
|
||||
return None
|
||||
info = cropper._compute_crop_info(image, image_path)
|
||||
return info
|
||||
|
||||
|
||||
def _load_refuge_model(device: str) -> Optional[RefugeUNet]:
|
||||
if not REFUGE_SEG_WEIGHTS.exists():
|
||||
return None
|
||||
model = RefugeUNet()
|
||||
try:
|
||||
state = torch.load(REFUGE_SEG_WEIGHTS, map_location=device)
|
||||
except Exception:
|
||||
return None
|
||||
state_dict = state.get("model", state) if isinstance(state, dict) else state
|
||||
try:
|
||||
model.load_state_dict(state_dict)
|
||||
except Exception:
|
||||
return None
|
||||
model.to(device)
|
||||
model.eval()
|
||||
return model
|
||||
|
||||
|
||||
def _refuge_seg_info(
|
||||
model: RefugeUNet, device: str, image_path: Path
|
||||
) -> Optional[Dict[str, float]]:
|
||||
try:
|
||||
image = Image.open(image_path).convert("RGB")
|
||||
except Exception:
|
||||
return None
|
||||
original_size = image.size
|
||||
image_resized = image.resize((512, 512), Image.BILINEAR)
|
||||
tensor = transforms.ToTensor()(image_resized).unsqueeze(0).to(device)
|
||||
with torch.no_grad():
|
||||
logits = model(tensor)
|
||||
mask = torch.sigmoid(logits)[0, 0]
|
||||
mask_np = (mask.cpu().numpy() > 0.5).astype(np.float32)
|
||||
mask_img = Image.fromarray(mask_np)
|
||||
mask_img = mask_img.resize(original_size, Image.NEAREST)
|
||||
mask_np = np.array(mask_img, dtype=np.float32)
|
||||
coords = np.argwhere(mask_np > 0.5)
|
||||
if coords.size == 0:
|
||||
return None
|
||||
ys, xs = coords[:, 0], coords[:, 1]
|
||||
centre_x = float(xs.mean())
|
||||
centre_y = float(ys.mean())
|
||||
width = float(xs.max() - xs.min())
|
||||
height = float(ys.max() - ys.min())
|
||||
diameter = max(width, height)
|
||||
radius = diameter / 2.0
|
||||
crop_radius = radius * SCALE
|
||||
left = max(0.0, centre_x - crop_radius)
|
||||
upper = max(0.0, centre_y - crop_radius)
|
||||
right = min(float(image.width), centre_x + crop_radius)
|
||||
lower = min(float(image.height), centre_y + crop_radius)
|
||||
return {
|
||||
"left": left,
|
||||
"upper": upper,
|
||||
"right": right,
|
||||
"lower": lower,
|
||||
}
|
||||
|
||||
|
||||
def _diff_bounds(cache: Dict[str, np.ndarray], gt: Dict[str, float]) -> Optional[float]:
|
||||
keys = ("left", "upper", "right", "lower")
|
||||
if not all(k in cache for k in keys):
|
||||
return None
|
||||
diffs = [abs(float(cache[k]) - float(gt[k])) for k in keys]
|
||||
return float(max(diffs))
|
||||
|
||||
|
||||
def _diff_features(
|
||||
cache: Dict[str, np.ndarray], gt: Dict[str, float]
|
||||
) -> Optional[float]:
|
||||
if "features" not in cache or "features" not in gt:
|
||||
return None
|
||||
cf = np.asarray(cache["features"], dtype=float).ravel()
|
||||
gf = np.asarray(gt["features"], dtype=float).ravel()
|
||||
if cf.shape != gf.shape:
|
||||
return None
|
||||
return float(np.max(np.abs(cf - gf)))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if not CACHE_DIR.exists():
|
||||
raise SystemExit(f"Cache dir not found: {CACHE_DIR}")
|
||||
if not MANIFEST_PATH.exists():
|
||||
raise SystemExit(f"Manifest not found: {MANIFEST_PATH}")
|
||||
|
||||
cache_files = sorted(CACHE_DIR.glob(f"*_s{int(SCALE * 100)}.npz"))
|
||||
if MAX_SAMPLES is not None:
|
||||
cache_files = cache_files[:MAX_SAMPLES]
|
||||
print(f"[debug] cache files found: {len(cache_files)}")
|
||||
|
||||
try:
|
||||
manifest_df = pd.read_csv(MANIFEST_PATH)
|
||||
except Exception as exc:
|
||||
raise SystemExit(f"Failed to read manifest: {exc}")
|
||||
manifest_images = manifest_df.get("image_path")
|
||||
if manifest_images is None:
|
||||
raise SystemExit("Manifest is missing image_path column.")
|
||||
manifest_images = manifest_images.dropna().astype(str)
|
||||
manifest_stems = {Path(p).stem for p in manifest_images}
|
||||
print(f"[debug] manifest image_path count: {len(manifest_images)}")
|
||||
print(f"[debug] manifest unique stems: {len(manifest_stems)}")
|
||||
|
||||
cache_stems = {_parse_stem(p) for p in cache_files}
|
||||
overlap = cache_stems & manifest_stems
|
||||
print(
|
||||
f"[debug] cache stems: {len(cache_stems)} overlap with manifest stems: {len(overlap)}"
|
||||
)
|
||||
if cache_files:
|
||||
print(f"[debug] example cache stems: {sorted(list(cache_stems))[:5]}")
|
||||
if manifest_stems:
|
||||
print(f"[debug] example manifest stems: {sorted(list(manifest_stems))[:5]}")
|
||||
|
||||
cropper = ManifestImageCropper(
|
||||
manifest_path=MANIFEST_PATH,
|
||||
scale=SCALE,
|
||||
target_size=224,
|
||||
cache_dir=None,
|
||||
)
|
||||
|
||||
rows: List[Dict[str, object]] = []
|
||||
for cache_path in cache_files:
|
||||
cache = _load_cache(cache_path)
|
||||
if cache is None:
|
||||
continue
|
||||
stem = _parse_stem(cache_path)
|
||||
image_path = _image_path_from_stem(stem)
|
||||
if image_path is None:
|
||||
continue
|
||||
|
||||
gt = _gt_info(cropper, image_path)
|
||||
if gt is None:
|
||||
continue
|
||||
|
||||
bounds_diff = _diff_bounds(cache, gt)
|
||||
feat_diff = _diff_features(cache, gt)
|
||||
|
||||
rows.append(
|
||||
{
|
||||
"file": cache_path.name,
|
||||
"bounds_diff": bounds_diff,
|
||||
"features_diff": feat_diff,
|
||||
"bounds_match": bounds_diff is not None and bounds_diff <= TOL_BOUNDS,
|
||||
"features_match": feat_diff is not None and feat_diff <= TOL_FEATURES,
|
||||
}
|
||||
)
|
||||
|
||||
if not rows:
|
||||
print("[warn] No cache entries matched GT manifest entries.")
|
||||
else:
|
||||
df = pd.DataFrame(rows)
|
||||
print(df.head(10).to_string(index=False))
|
||||
print("\nSummary:")
|
||||
print(df[["bounds_diff", "features_diff"]].describe().to_string())
|
||||
if df["bounds_match"].notna().any():
|
||||
match_rate = df["bounds_match"].mean()
|
||||
print(f"\nBounds match rate (<= {TOL_BOUNDS}px): {match_rate:.3f}")
|
||||
if df["features_match"].notna().any():
|
||||
match_rate = df["features_match"].mean()
|
||||
print(f"Features match rate (<= {TOL_FEATURES}): {match_rate:.3f}")
|
||||
|
||||
print("\nUNet variant comparisons (no cache writes):")
|
||||
for name, weights, normalize in UNET_VARIANTS:
|
||||
if not weights.exists():
|
||||
print(f"[warn] {name}: weights not found at {weights}")
|
||||
continue
|
||||
|
||||
unet = UNetImageCropper(
|
||||
manifest_path=MANIFEST_PATH,
|
||||
weights_path=weights,
|
||||
normalize=normalize,
|
||||
threshold=0.5,
|
||||
tta=False,
|
||||
scale=SCALE,
|
||||
target_size=224,
|
||||
cache_dir=None, # ensure no cache writes
|
||||
)
|
||||
|
||||
u_rows: List[Dict[str, object]] = []
|
||||
missing_images = 0
|
||||
unet_none = 0
|
||||
cache_missing = 0
|
||||
exceptions = 0
|
||||
for cache_path in cache_files:
|
||||
cache = _load_cache(cache_path)
|
||||
if cache is None:
|
||||
cache_missing += 1
|
||||
continue
|
||||
stem = _parse_stem(cache_path)
|
||||
image_path = _image_path_from_stem(stem)
|
||||
if image_path is None:
|
||||
missing_images += 1
|
||||
continue
|
||||
try:
|
||||
info = _unet_info(unet, image_path)
|
||||
except Exception:
|
||||
exceptions += 1
|
||||
continue
|
||||
if info is None:
|
||||
unet_none += 1
|
||||
continue
|
||||
bounds_diff = _diff_bounds(cache, info)
|
||||
feat_diff = _diff_features(cache, info)
|
||||
u_rows.append(
|
||||
{
|
||||
"bounds_diff": bounds_diff,
|
||||
"features_diff": feat_diff,
|
||||
"bounds_match": bounds_diff is not None
|
||||
and bounds_diff <= TOL_BOUNDS,
|
||||
"features_match": feat_diff is not None
|
||||
and feat_diff <= TOL_FEATURES,
|
||||
}
|
||||
)
|
||||
|
||||
if not u_rows:
|
||||
print(
|
||||
f"[warn] {name}: no comparisons computed "
|
||||
f"(cache_missing={cache_missing}, missing_images={missing_images}, "
|
||||
f"unet_none={unet_none}, exceptions={exceptions})"
|
||||
)
|
||||
continue
|
||||
u_df = pd.DataFrame(u_rows)
|
||||
b_mean = float(u_df["bounds_diff"].mean())
|
||||
f_mean = float(u_df["features_diff"].mean())
|
||||
b_match = float(u_df["bounds_match"].mean())
|
||||
f_match = float(u_df["features_match"].mean())
|
||||
|
||||
print(
|
||||
f"{name}: mean bounds diff={b_mean:.3f}, mean feat diff={f_mean:.6f}, "
|
||||
f"bounds match rate={b_match:.3f}, features match rate={f_match:.3f}"
|
||||
)
|
||||
|
||||
print("\nRefuge segmentation model comparison (bounds only, no cache writes):")
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
refuge_model = _load_refuge_model(device)
|
||||
if refuge_model is None:
|
||||
print(f"[warn] refuge_segmentation_best.pt not found or failed to load at {REFUGE_SEG_WEIGHTS}")
|
||||
return
|
||||
|
||||
r_rows: List[Dict[str, object]] = []
|
||||
missing_images = 0
|
||||
cache_missing = 0
|
||||
model_none = 0
|
||||
exceptions = 0
|
||||
for cache_path in cache_files:
|
||||
cache = _load_cache(cache_path)
|
||||
if cache is None:
|
||||
cache_missing += 1
|
||||
continue
|
||||
stem = _parse_stem(cache_path)
|
||||
image_path = _image_path_from_stem(stem)
|
||||
if image_path is None:
|
||||
missing_images += 1
|
||||
continue
|
||||
try:
|
||||
info = _refuge_seg_info(refuge_model, device, image_path)
|
||||
except Exception:
|
||||
exceptions += 1
|
||||
continue
|
||||
if info is None:
|
||||
model_none += 1
|
||||
continue
|
||||
bounds_diff = _diff_bounds(cache, info)
|
||||
r_rows.append(
|
||||
{
|
||||
"bounds_diff": bounds_diff,
|
||||
"bounds_match": bounds_diff is not None and bounds_diff <= TOL_BOUNDS,
|
||||
}
|
||||
)
|
||||
|
||||
if not r_rows:
|
||||
print(
|
||||
"[warn] refuge_segmentation_best: no comparisons computed "
|
||||
f"(cache_missing={cache_missing}, missing_images={missing_images}, "
|
||||
f"model_none={model_none}, exceptions={exceptions})"
|
||||
)
|
||||
return
|
||||
|
||||
r_df = pd.DataFrame(r_rows)
|
||||
b_mean = float(r_df["bounds_diff"].mean())
|
||||
b_match = float(r_df["bounds_match"].mean())
|
||||
print(
|
||||
f"refuge_segmentation_best: mean bounds diff={b_mean:.3f}, "
|
||||
f"bounds match rate={b_match:.3f}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+249
@@ -0,0 +1,249 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare suspect AUC from image tower vs crop-derived geometry (CDR)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from PIL import Image
|
||||
from sklearn.metrics import roc_auc_score
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from classes import build_papila_clinical
|
||||
from classes.hypertower import UNetImageCropper, ManifestImageCropper
|
||||
|
||||
|
||||
# ---------------------------
|
||||
# Config (edit in IDE)
|
||||
# ---------------------------
|
||||
RUN_DIRS = [
|
||||
Path("analysis_data/1030_Balanced_Unet_Perimg_Resnet_SE16NormB_SE16NormT_multi_fused/1030_Balanced_Unet_Perimg_Resnet_SE16NormB_SE16NormT_multi_fused_20251030_091842"),
|
||||
Path("analysis_data/1030_Balanced_GT_Perimg_Resnet_SE16NormB_SE16NormT_multi_fused/1030_Balanced_GT_Perimg_Resnet_SE16NormB_SE16NormT_multi_fused_20251030_113730"),
|
||||
]
|
||||
GEOM_CACHE_ROOT = Path("analysis_data/geometry_cache")
|
||||
SUSPECT_LABEL = 2
|
||||
|
||||
|
||||
def _load_json(path: Path) -> Dict:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(path.read_text())
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _drop_holdout_rows(clinical, holdout_path: Path) -> None:
|
||||
if not holdout_path.exists():
|
||||
return
|
||||
holdout = pd.read_csv(holdout_path)
|
||||
if holdout.empty:
|
||||
return
|
||||
if "Patient ID" not in holdout.columns or "eyeID" not in holdout.columns:
|
||||
return
|
||||
holdout_keys = set(zip(holdout["Patient ID"].astype(int), holdout["eyeID"].astype(str)))
|
||||
df = clinical.df.copy()
|
||||
df["_key"] = list(zip(df["Patient ID"].astype(int), df["eyeID"].astype(str)))
|
||||
df = df[~df["_key"].isin(holdout_keys)].drop(columns=["_key"]).reset_index(drop=True)
|
||||
|
||||
clinical.frames = [df.copy()]
|
||||
clinical.df = df.copy()
|
||||
clinical._infer_or_validate_feature_types()
|
||||
clinical._compute_numeric_stats()
|
||||
clinical._build_cat_maps()
|
||||
clinical._compute_feature_dim()
|
||||
clinical._build_kfold_indices()
|
||||
|
||||
|
||||
def _make_cropper(args: Dict, cache_dir: Path):
|
||||
manifest = args.get("img_crop_manifest")
|
||||
if not manifest:
|
||||
raise RuntimeError("img_crop_manifest missing; cannot compute geometry features.")
|
||||
|
||||
scale = float(args.get("img_crop_scale", 2.5))
|
||||
target_size = int(args.get("img_crop_size", 224))
|
||||
use_gt = bool(args.get("img_crop_gt", False))
|
||||
|
||||
if use_gt:
|
||||
return ManifestImageCropper(
|
||||
manifest_path=Path(manifest),
|
||||
scale=scale,
|
||||
target_size=target_size,
|
||||
cache_dir=cache_dir,
|
||||
)
|
||||
|
||||
weights = args.get("img_crop_weights")
|
||||
if not weights:
|
||||
raise RuntimeError("img_crop_weights missing for UNet cropper.")
|
||||
|
||||
normalize = args.get("img_crop_normalize", "per_image")
|
||||
threshold = float(args.get("img_crop_threshold", 0.5))
|
||||
tta = bool(args.get("img_crop_tta", False))
|
||||
return UNetImageCropper(
|
||||
manifest_path=Path(manifest),
|
||||
weights_path=Path(weights),
|
||||
normalize=normalize,
|
||||
threshold=threshold,
|
||||
tta=tta,
|
||||
scale=scale,
|
||||
target_size=target_size,
|
||||
cache_dir=cache_dir,
|
||||
)
|
||||
|
||||
|
||||
def _geometry_scores(
|
||||
clinical,
|
||||
cropper,
|
||||
test_df: pd.DataFrame,
|
||||
) -> Tuple[np.ndarray, np.ndarray]:
|
||||
scores: List[float] = []
|
||||
keep_mask: List[bool] = []
|
||||
for _, row in test_df.iterrows():
|
||||
img_path = clinical.get_image_path(row)
|
||||
try:
|
||||
image = Image.open(img_path).convert("RGB")
|
||||
except Exception:
|
||||
scores.append(float("nan"))
|
||||
keep_mask.append(False)
|
||||
continue
|
||||
feats = cropper.geometry_features(image, img_path)
|
||||
if feats is None or len(feats) == 0:
|
||||
scores.append(float("nan"))
|
||||
keep_mask.append(False)
|
||||
else:
|
||||
scores.append(float(feats[0])) # area_ratio (CDR)
|
||||
keep_mask.append(True)
|
||||
return np.asarray(scores, dtype=float), np.asarray(keep_mask, dtype=bool)
|
||||
|
||||
|
||||
def _suspect_auc(y_true: np.ndarray, scores: np.ndarray) -> float:
|
||||
y = (y_true == SUSPECT_LABEL).astype(int)
|
||||
if y.sum() == 0 or y.sum() == len(y):
|
||||
return float("nan")
|
||||
return float(roc_auc_score(y, scores))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rows: List[Dict[str, object]] = []
|
||||
|
||||
for run_dir in RUN_DIRS:
|
||||
cli_path = run_dir / "cli_args.json"
|
||||
cli_args = _load_json(cli_path)
|
||||
if not cli_args:
|
||||
print(f"[warn] Missing cli_args.json in {run_dir}")
|
||||
continue
|
||||
|
||||
label_col = cli_args.get("label_col", "Diagnosis")
|
||||
cat_cols = cli_args.get("cat_cols", ["Gender", "Phakic/Pseudophakic"])
|
||||
n_splits = int(cli_args.get("n_splits", 5))
|
||||
fold_seed = int(cli_args.get("fold_seed", 42))
|
||||
eval_mode = str(cli_args.get("eval_mode", "multiclass")).lower()
|
||||
|
||||
clinical = build_papila_clinical(
|
||||
image_dir=cli_args.get("image_dir", "Papila/FundusImages"),
|
||||
clinical_dir=cli_args.get("clinical_dir", "Papila/ClinicalData"),
|
||||
label_col=label_col,
|
||||
cat_cols=cat_cols,
|
||||
n_splits=n_splits,
|
||||
random_seed=fold_seed,
|
||||
)
|
||||
|
||||
if eval_mode == "binary":
|
||||
clinical.df = clinical.df[clinical.df[label_col].isin([0, 1])].reset_index(drop=True)
|
||||
clinical.frames = [clinical.df.copy()]
|
||||
clinical._infer_or_validate_feature_types()
|
||||
clinical._compute_numeric_stats()
|
||||
clinical._build_cat_maps()
|
||||
clinical._compute_feature_dim()
|
||||
clinical._build_kfold_indices()
|
||||
|
||||
_drop_holdout_rows(clinical, run_dir / "holdout.csv")
|
||||
|
||||
cache_dir = GEOM_CACHE_ROOT / run_dir.name
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
cropper = _make_cropper(cli_args, cache_dir=cache_dir)
|
||||
|
||||
all_geom_scores: List[float] = []
|
||||
all_img_scores: List[float] = []
|
||||
all_y: List[int] = []
|
||||
|
||||
for fold in range(n_splits):
|
||||
y_path = run_dir / f"fold{fold}_y_true.npy"
|
||||
p_img_path = run_dir / f"fold{fold}_probs_img.npy"
|
||||
if not y_path.exists() or not p_img_path.exists():
|
||||
continue
|
||||
|
||||
y_true = np.load(y_path)
|
||||
probs_img = np.load(p_img_path)
|
||||
if probs_img.ndim != 2 or probs_img.shape[1] <= SUSPECT_LABEL:
|
||||
continue
|
||||
|
||||
_, test_df = clinical.get_split_dfs(fold)
|
||||
if len(test_df) != len(y_true):
|
||||
print(
|
||||
f"[warn] {run_dir.name} fold{fold}: test_df len {len(test_df)} != y_true len {len(y_true)}"
|
||||
)
|
||||
|
||||
geom_scores, keep_mask = _geometry_scores(clinical, cropper, test_df)
|
||||
if keep_mask.sum() == 0:
|
||||
print(f"[warn] {run_dir.name} fold{fold}: no valid geometry features")
|
||||
continue
|
||||
|
||||
y_fold = y_true[: len(geom_scores)][keep_mask]
|
||||
geom_fold = geom_scores[keep_mask]
|
||||
img_fold = probs_img[: len(geom_scores), SUSPECT_LABEL][keep_mask]
|
||||
|
||||
geom_auc = _suspect_auc(y_fold, geom_fold)
|
||||
img_auc = _suspect_auc(y_fold, img_fold)
|
||||
|
||||
rows.append(
|
||||
{
|
||||
"run": run_dir.name,
|
||||
"fold": fold,
|
||||
"metric": "suspect_auc",
|
||||
"image_auc": img_auc,
|
||||
"geometry_auc": geom_auc,
|
||||
"n": int(len(y_fold)),
|
||||
}
|
||||
)
|
||||
|
||||
all_geom_scores.append(geom_fold)
|
||||
all_img_scores.append(img_fold)
|
||||
all_y.append(y_fold)
|
||||
|
||||
if all_y:
|
||||
y_all = np.concatenate(all_y)
|
||||
geom_all = np.concatenate(all_geom_scores)
|
||||
img_all = np.concatenate(all_img_scores)
|
||||
rows.append(
|
||||
{
|
||||
"run": run_dir.name,
|
||||
"fold": "all",
|
||||
"metric": "suspect_auc",
|
||||
"image_auc": _suspect_auc(y_all, img_all),
|
||||
"geometry_auc": _suspect_auc(y_all, geom_all),
|
||||
"n": int(len(y_all)),
|
||||
}
|
||||
)
|
||||
|
||||
if not rows:
|
||||
raise SystemExit("No results produced; check run paths and files.")
|
||||
|
||||
df = pd.DataFrame(rows)
|
||||
out_path = GEOM_CACHE_ROOT / "suspect_auc_geometry_vs_image.csv"
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
df.to_csv(out_path, index=False)
|
||||
print(df.to_string(index=False, float_format=lambda x: f"{x:.4f}"))
|
||||
print(f"\nSaved: {out_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+1826
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,314 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Shared helpers for building grid search analytics.
|
||||
|
||||
The class below will gradually accumulate reusable utilities for working with
|
||||
grid search outputs (summary.json, cli_args.json, etc).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import csv
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, Iterator, List, Optional
|
||||
|
||||
import numpy as np
|
||||
|
||||
DEFAULT_EXCLUDE_KEYS = {
|
||||
"run_id",
|
||||
"fold_metrics",
|
||||
"best_metric",
|
||||
"best_metric_mode",
|
||||
"best_metric_mean",
|
||||
"best_metric_std",
|
||||
"eval_mode",
|
||||
"n_splits",
|
||||
"num_classes",
|
||||
}
|
||||
|
||||
|
||||
class GridSearchAnalytics:
|
||||
"""Utility wrapper for inspecting grid search result directories."""
|
||||
|
||||
def __init__(self,
|
||||
analysis_dir: Path | str,
|
||||
exclude_keys: Optional[Iterable[str]] = None) -> None:
|
||||
self.analysis_dir = Path(analysis_dir)
|
||||
if not self.analysis_dir.exists():
|
||||
raise FileNotFoundError(f"analysis_dir does not exist: {self.analysis_dir}")
|
||||
self.exclude_keys = set(exclude_keys or DEFAULT_EXCLUDE_KEYS)
|
||||
|
||||
def iter_run_dirs(self, shallow: bool = True) -> Iterator[Path]:
|
||||
"""
|
||||
Yield run directories containing grid search artifacts.
|
||||
|
||||
Shallow iteration only walks direct children. Deep iteration scans the
|
||||
entire subtree.
|
||||
"""
|
||||
candidates: Iterable[Path]
|
||||
if shallow:
|
||||
candidates = (p for p in sorted(self.analysis_dir.iterdir()) if p.is_dir())
|
||||
else:
|
||||
candidates = (p for p in self.analysis_dir.rglob("*") if p.is_dir())
|
||||
for run_dir in candidates:
|
||||
summary = run_dir / "summary.json"
|
||||
cli = run_dir / "cli_args.json"
|
||||
if summary.exists() or cli.exists():
|
||||
yield run_dir
|
||||
|
||||
def read_summary(self, run_dir: Path) -> Optional[Dict[str, object]]:
|
||||
"""Load summary.json for a run directory."""
|
||||
return self._read_json(run_dir / "summary.json")
|
||||
|
||||
def read_cli_args(self, run_dir: Path) -> Optional[Dict[str, object]]:
|
||||
"""Load cli_args.json for a run directory."""
|
||||
return self._read_json(run_dir / "cli_args.json")
|
||||
|
||||
def read_run_id(self,
|
||||
run_dir: Path,
|
||||
summary: Optional[Dict[str, object]]) -> str:
|
||||
if summary:
|
||||
rid = summary.get("run_id")
|
||||
if isinstance(rid, str) and rid:
|
||||
return rid
|
||||
return run_dir.name
|
||||
|
||||
def task_from_summary(self, summary: Optional[Dict[str, object]]) -> Optional[str]:
|
||||
"""Infer task (binary vs multiclass) from a summary payload."""
|
||||
if not summary:
|
||||
return None
|
||||
eval_mode = summary.get("eval_mode")
|
||||
if isinstance(eval_mode, str):
|
||||
mode = eval_mode.strip().lower()
|
||||
if mode == "binary":
|
||||
return "binary"
|
||||
if mode in {"multiclass", "multi", "multi-class"}:
|
||||
return "multiclass"
|
||||
num_classes = summary.get("num_classes")
|
||||
if isinstance(num_classes, (int, float)):
|
||||
return "binary" if int(num_classes) <= 2 else "multiclass"
|
||||
return None
|
||||
|
||||
def flatten_config(self,
|
||||
data: Dict[str, object],
|
||||
prefix: str = "",
|
||||
exclude_keys: Optional[Iterable[str]] = None) -> Dict[str, object]:
|
||||
"""Flatten nested CLI args or config dictionaries for analysis."""
|
||||
out: Dict[str, object] = {}
|
||||
excludes = set(exclude_keys or self.exclude_keys)
|
||||
for key, value in data.items():
|
||||
if key in excludes or key.startswith("best_"):
|
||||
continue
|
||||
full_key = f"{prefix}{key}" if not prefix else f"{prefix}.{key}"
|
||||
if isinstance(value, dict):
|
||||
out.update(self.flatten_config(value, full_key, exclude_keys=excludes))
|
||||
continue
|
||||
if isinstance(value, list):
|
||||
continue
|
||||
out[full_key] = value
|
||||
return out
|
||||
|
||||
def fusion_correction_events(self,
|
||||
output_csv: Path | str | None = None,
|
||||
shallow: bool = True) -> Path:
|
||||
"""
|
||||
Build a table of cases where the fused head is correct while both towers
|
||||
are wrong. Rows are written to CSV for downstream analysis.
|
||||
"""
|
||||
output_path = Path(output_csv) if output_csv else Path("analysis_data/grid_search_analytics/fusion_corrections.csv")
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
rows: List[Dict[str, object]] = []
|
||||
for run_dir in self.iter_run_dirs(shallow=shallow):
|
||||
summary = self.read_summary(run_dir)
|
||||
run_id = self.read_run_id(run_dir, summary)
|
||||
folds = self._available_folds(run_dir, summary)
|
||||
for fold in folds:
|
||||
y_true = self._load_y_true(run_dir, fold)
|
||||
if y_true is None:
|
||||
continue
|
||||
epoch_prob_paths = self._collect_epoch_prob_paths(run_dir, fold)
|
||||
if not epoch_prob_paths:
|
||||
# Per-epoch dumps were not found; fall back to the saved fold-level probabilities.
|
||||
base_paths = self._collect_base_prob_paths(run_dir, fold)
|
||||
if base_paths:
|
||||
epoch_hint = self._fold_epoch_hint(summary, fold)
|
||||
epoch_prob_paths = {epoch_hint if epoch_hint is not None else 0: base_paths}
|
||||
for epoch, paths in epoch_prob_paths.items():
|
||||
arrays = {head: self._load_probs_array(path) for head, path in paths.items()}
|
||||
if not self._has_all_heads(arrays):
|
||||
continue
|
||||
events = self._fusion_corrections_for_probs(y_true, arrays, run_id, fold, epoch)
|
||||
rows.extend(events)
|
||||
|
||||
if rows:
|
||||
fieldnames = [
|
||||
"run_id",
|
||||
"fold",
|
||||
"epoch",
|
||||
"index",
|
||||
"y_true",
|
||||
"pred_fused",
|
||||
"pred_img",
|
||||
"pred_md",
|
||||
"conf_fused",
|
||||
"conf_img",
|
||||
"conf_md",
|
||||
]
|
||||
with output_path.open("w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
else:
|
||||
output_path.write_text("")
|
||||
return output_path
|
||||
|
||||
@staticmethod
|
||||
def _read_json(path: Path) -> Optional[Dict[str, object]]:
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
data = json.loads(path.read_text())
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _fold_epoch_hint(summary: Optional[Dict[str, object]], fold: int) -> Optional[int]:
|
||||
if not summary:
|
||||
return None
|
||||
fold_metrics = summary.get("fold_metrics") or []
|
||||
for entry in fold_metrics:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
if entry.get("fold") == fold:
|
||||
stats = entry.get("stats") if isinstance(entry.get("stats"), dict) else {}
|
||||
epoch = stats.get("epoch") or entry.get("best_epoch")
|
||||
if isinstance(epoch, (int, float)):
|
||||
return int(epoch)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _available_folds(run_dir: Path, summary: Optional[Dict[str, object]]) -> List[int]:
|
||||
folds: List[int] = []
|
||||
if summary:
|
||||
for entry in summary.get("fold_metrics") or []:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
fold_idx = entry.get("fold")
|
||||
if isinstance(fold_idx, int):
|
||||
folds.append(fold_idx)
|
||||
if not folds:
|
||||
pattern = re.compile(r"fold(\d+)_y_true\.npy$")
|
||||
for path in run_dir.glob("fold*_y_true.npy"):
|
||||
match = pattern.match(path.name)
|
||||
if match:
|
||||
folds.append(int(match.group(1)))
|
||||
return sorted(set(folds))
|
||||
|
||||
@staticmethod
|
||||
def _load_y_true(run_dir: Path, fold: int) -> Optional[np.ndarray]:
|
||||
path = run_dir / f"fold{fold}_y_true.npy"
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
return np.load(path)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _collect_epoch_prob_paths(run_dir: Path, fold: int) -> Dict[int, Dict[str, Path]]:
|
||||
pattern = re.compile(rf"fold{fold}_epoch(\d+)_probs_(\w+)\.npy$")
|
||||
epoch_paths: Dict[int, Dict[str, Path]] = {}
|
||||
for path in run_dir.glob(f"fold{fold}_epoch*_probs_*.npy"):
|
||||
match = pattern.match(path.name)
|
||||
if not match:
|
||||
continue
|
||||
epoch = int(match.group(1))
|
||||
head = match.group(2)
|
||||
epoch_paths.setdefault(epoch, {})[head] = path
|
||||
return epoch_paths
|
||||
|
||||
@staticmethod
|
||||
def _collect_base_prob_paths(run_dir: Path, fold: int) -> Dict[str, Path]:
|
||||
paths: Dict[str, Path] = {}
|
||||
for head in ("fused", "img", "md"):
|
||||
candidate = run_dir / f"fold{fold}_probs_{head}.npy"
|
||||
if candidate.exists():
|
||||
paths[head] = candidate
|
||||
return paths
|
||||
|
||||
@staticmethod
|
||||
def _load_probs_array(path: Path) -> Optional[np.ndarray]:
|
||||
try:
|
||||
return np.load(path)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _prepare_probs(arr: np.ndarray) -> Optional[np.ndarray]:
|
||||
if arr is None:
|
||||
return None
|
||||
probs = np.asarray(arr, dtype=float)
|
||||
if probs.ndim == 1:
|
||||
probs = np.stack([1.0 - probs, probs], axis=1)
|
||||
if probs.ndim != 2:
|
||||
return None
|
||||
return probs
|
||||
|
||||
@staticmethod
|
||||
def _has_all_heads(arrays: Dict[str, Optional[np.ndarray]]) -> bool:
|
||||
needed = ("fused", "img", "md")
|
||||
return all(arrays.get(head) is not None for head in needed)
|
||||
|
||||
def _fusion_corrections_for_probs(self,
|
||||
y_true: np.ndarray,
|
||||
arrays: Dict[str, np.ndarray],
|
||||
run_id: str,
|
||||
fold: int,
|
||||
epoch: int) -> List[Dict[str, object]]:
|
||||
fused = self._prepare_probs(arrays.get("fused"))
|
||||
img = self._prepare_probs(arrays.get("img"))
|
||||
md = self._prepare_probs(arrays.get("md"))
|
||||
if fused is None or img is None or md is None:
|
||||
return []
|
||||
if not (len(fused) == len(img) == len(md) == len(y_true)):
|
||||
return []
|
||||
|
||||
fused_pred = fused.argmax(axis=1)
|
||||
img_pred = img.argmax(axis=1)
|
||||
md_pred = md.argmax(axis=1)
|
||||
|
||||
fused_conf = np.take_along_axis(fused, fused_pred[:, None], axis=1).squeeze(1)
|
||||
img_conf = np.take_along_axis(img, img_pred[:, None], axis=1).squeeze(1)
|
||||
md_conf = np.take_along_axis(md, md_pred[:, None], axis=1).squeeze(1)
|
||||
|
||||
mask = (fused_pred == y_true) & (img_pred != y_true) & (md_pred != y_true)
|
||||
indices = np.nonzero(mask)[0]
|
||||
|
||||
events: List[Dict[str, object]] = []
|
||||
for idx in indices:
|
||||
events.append({
|
||||
"run_id": run_id,
|
||||
"fold": fold,
|
||||
"epoch": epoch,
|
||||
"index": int(idx),
|
||||
"y_true": int(y_true[idx]),
|
||||
"pred_fused": int(fused_pred[idx]),
|
||||
"pred_img": int(img_pred[idx]),
|
||||
"pred_md": int(md_pred[idx]),
|
||||
"conf_fused": float(fused_conf[idx]),
|
||||
"conf_img": float(img_conf[idx]),
|
||||
"conf_md": float(md_conf[idx]),
|
||||
})
|
||||
return events
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
analytics = GridSearchAnalytics(Path("analysis_data/grid_search"))
|
||||
output = analytics.fusion_correction_events()
|
||||
print(f"Fusion correction events written to {output}")
|
||||
@@ -0,0 +1,729 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Build an HTML heatmap-style grid for grid search runs.
|
||||
|
||||
Each column is a run. The header shows mean metrics (auc, acc, holdout_auc,
|
||||
holdout_acc). Rows encode hyperparameter options as red/green boxes.
|
||||
|
||||
Example:
|
||||
python scripts/grid_search_analytics/grid_search_heatmap.py \
|
||||
--analysis-dir analysis_data/grid_search \
|
||||
--task binary \
|
||||
--sort-by holdout_auc --desc \
|
||||
--top 40 \
|
||||
--format plot \
|
||||
--output analysis_data/grid_search_heatmap.png
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from html import escape
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
DEFAULT_EXCLUDE_KEYS = {
|
||||
"run_id",
|
||||
"fold_metrics",
|
||||
"best_metric",
|
||||
"best_metric_mode",
|
||||
"best_metric_mean",
|
||||
"best_metric_std",
|
||||
"eval_mode",
|
||||
"n_splits",
|
||||
"num_classes",
|
||||
}
|
||||
|
||||
|
||||
def to_float(value: Optional[object]) -> Optional[float]:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
num = float(value)
|
||||
if math.isnan(num):
|
||||
return None
|
||||
return num
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
value = value.strip()
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
num = float(value)
|
||||
except ValueError:
|
||||
return None
|
||||
if math.isnan(num):
|
||||
return None
|
||||
return num
|
||||
|
||||
|
||||
def mean(values: List[float]) -> Optional[float]:
|
||||
return (sum(values) / len(values)) if values else None
|
||||
|
||||
|
||||
def read_summary(run_dir: Path) -> Optional[Dict[str, object]]:
|
||||
summary_path = run_dir / "summary.json"
|
||||
if not summary_path.exists():
|
||||
return None
|
||||
try:
|
||||
data = json.loads(summary_path.read_text())
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
def read_cli_args(run_dir: Path) -> Optional[Dict[str, object]]:
|
||||
cli_path = run_dir / "cli_args.json"
|
||||
if not cli_path.exists():
|
||||
return None
|
||||
try:
|
||||
data = json.loads(cli_path.read_text())
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
def read_run_id(run_dir: Path, summary: Optional[Dict[str, object]]) -> str:
|
||||
if summary:
|
||||
rid = summary.get("run_id")
|
||||
if isinstance(rid, str) and rid:
|
||||
return rid
|
||||
return run_dir.name
|
||||
|
||||
|
||||
def task_from_summary(summary: Optional[Dict[str, object]]) -> Optional[str]:
|
||||
if not summary:
|
||||
return None
|
||||
eval_mode = summary.get("eval_mode")
|
||||
if isinstance(eval_mode, str):
|
||||
mode = eval_mode.strip().lower()
|
||||
if mode == "binary":
|
||||
return "binary"
|
||||
if mode in {"multiclass", "multi", "multi-class"}:
|
||||
return "multiclass"
|
||||
num_classes = summary.get("num_classes")
|
||||
if isinstance(num_classes, (int, float)):
|
||||
return "binary" if int(num_classes) <= 2 else "multiclass"
|
||||
return None
|
||||
|
||||
|
||||
def metric_from_stats(stats: Dict[str, object], metric: str) -> Optional[float]:
|
||||
if metric.startswith("holdout_") and stats.get("holdout_best_monitor") == metric:
|
||||
best_val = to_float(stats.get("holdout_best_so_far"))
|
||||
if best_val is not None:
|
||||
return best_val
|
||||
return to_float(stats.get(metric))
|
||||
|
||||
|
||||
def mean_metric(summary: Dict[str, object], metric: str) -> Optional[float]:
|
||||
folds = summary.get("fold_metrics") or []
|
||||
if not isinstance(folds, list) or not folds:
|
||||
return None
|
||||
values = []
|
||||
for fold in folds:
|
||||
stats = fold.get("stats") if isinstance(fold, dict) else None
|
||||
if not isinstance(stats, dict):
|
||||
return None
|
||||
val = metric_from_stats(stats, metric)
|
||||
if val is None:
|
||||
return None
|
||||
values.append(val)
|
||||
return mean(values)
|
||||
|
||||
|
||||
def flatten_config(data: Dict[str, object],
|
||||
prefix: str = "",
|
||||
exclude_keys: Optional[Iterable[str]] = None) -> Dict[str, object]:
|
||||
out: Dict[str, object] = {}
|
||||
excludes = set(exclude_keys or [])
|
||||
for key, value in data.items():
|
||||
if key in excludes or key.startswith("best_"):
|
||||
continue
|
||||
full_key = f"{prefix}{key}" if not prefix else f"{prefix}.{key}"
|
||||
if isinstance(value, dict):
|
||||
out.update(flatten_config(value, full_key, exclude_keys=excludes))
|
||||
continue
|
||||
if isinstance(value, list):
|
||||
continue
|
||||
out[full_key] = value
|
||||
return out
|
||||
|
||||
|
||||
def sort_value_key(value: object) -> Tuple[int, object]:
|
||||
if value is None:
|
||||
return (2, "")
|
||||
if isinstance(value, bool):
|
||||
return (0, int(value))
|
||||
if isinstance(value, (int, float)):
|
||||
return (0, value)
|
||||
return (1, str(value))
|
||||
|
||||
|
||||
def format_value(value: object) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if isinstance(value, int):
|
||||
return str(value)
|
||||
if isinstance(value, float):
|
||||
return f"{value:.6g}"
|
||||
return str(value)
|
||||
|
||||
|
||||
def format_metric(value: Optional[float]) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
return f"{value:.4f}"
|
||||
|
||||
|
||||
def render_progress(current: int, total: Optional[int], matched: int) -> str:
|
||||
if total:
|
||||
width = 30
|
||||
filled = int(width * current / total)
|
||||
bar = "#" * filled + "-" * (width - filled)
|
||||
return f"[{bar}] {current}/{total} matched {matched}"
|
||||
return f"Scanned {current} dirs, matched {matched}"
|
||||
|
||||
|
||||
def iter_run_dirs(root: Path, shallow: bool, show_progress: bool) -> Iterable[Path]:
|
||||
if shallow:
|
||||
entries = [entry for entry in root.iterdir() if entry.is_dir()]
|
||||
entries.sort(key=lambda p: p.name)
|
||||
total = len(entries)
|
||||
matched = 0
|
||||
last_update = 0.0
|
||||
for idx, entry in enumerate(entries, start=1):
|
||||
if show_progress:
|
||||
now = time.monotonic()
|
||||
if now - last_update >= 0.1 or idx == total:
|
||||
msg = render_progress(idx, total, matched)
|
||||
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
|
||||
last_update = now
|
||||
if (entry / "summary.json").is_file():
|
||||
matched += 1
|
||||
yield entry
|
||||
if show_progress:
|
||||
print(file=sys.stderr)
|
||||
return
|
||||
|
||||
matched = 0
|
||||
scanned = 0
|
||||
last_update = 0.0
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
scanned += 1
|
||||
if show_progress:
|
||||
now = time.monotonic()
|
||||
if now - last_update >= 0.2:
|
||||
msg = render_progress(scanned, None, matched)
|
||||
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
|
||||
last_update = now
|
||||
if "summary.json" in filenames:
|
||||
matched += 1
|
||||
yield Path(dirpath)
|
||||
if show_progress:
|
||||
msg = render_progress(scanned, None, matched)
|
||||
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
|
||||
print(file=sys.stderr)
|
||||
|
||||
|
||||
def build_html(runs: List[Dict[str, object]],
|
||||
row_specs: List[Tuple[str, object]],
|
||||
title: str,
|
||||
filters: List[str]) -> str:
|
||||
lines: List[str] = []
|
||||
lines.append("<!doctype html>")
|
||||
lines.append("<html lang=\"en\">")
|
||||
lines.append("<head>")
|
||||
lines.append("<meta charset=\"utf-8\">")
|
||||
lines.append(f"<title>{escape(title)}</title>")
|
||||
lines.append("<style>")
|
||||
lines.append(":root { --green: #4caf50; --red: #d9534f; --grid: #d0d0d0; --header: #f0f0f0; }")
|
||||
lines.append("body { margin: 0; padding: 16px; font-family: \"Courier New\", monospace; }")
|
||||
lines.append(".wrap { overflow-x: auto; }")
|
||||
lines.append("table { border-collapse: collapse; font-size: 12px; }")
|
||||
lines.append("th, td { border: 1px solid var(--grid); padding: 4px; text-align: center; }")
|
||||
lines.append("th.row-label { text-align: left; background: var(--header); position: sticky; left: 0; }")
|
||||
lines.append("thead th { background: var(--header); position: sticky; top: 0; z-index: 1; }")
|
||||
lines.append("th.run-id { writing-mode: vertical-rl; transform: rotate(180deg); white-space: nowrap; }")
|
||||
lines.append("td.cell { width: 14px; height: 14px; padding: 0; }")
|
||||
lines.append("td.on { background: var(--green); }")
|
||||
lines.append("td.off { background: var(--red); }")
|
||||
lines.append(".meta { margin-bottom: 12px; }")
|
||||
lines.append("</style>")
|
||||
lines.append("</head>")
|
||||
lines.append("<body>")
|
||||
lines.append(f"<h2>{escape(title)}</h2>")
|
||||
if filters:
|
||||
lines.append("<div class=\"meta\">")
|
||||
for item in filters:
|
||||
lines.append(f"<div>{escape(item)}</div>")
|
||||
lines.append("</div>")
|
||||
lines.append("<div class=\"wrap\">")
|
||||
lines.append("<table>")
|
||||
lines.append("<thead>")
|
||||
lines.append("<tr>")
|
||||
lines.append("<th>run_id</th>")
|
||||
for run in runs:
|
||||
run_id = escape(str(run.get("run_id", "")))
|
||||
rel_path = escape(str(run.get("relative_path", "")))
|
||||
title_attr = f" title=\"{rel_path}\"" if rel_path else ""
|
||||
lines.append(f"<th class=\"run-id\"{title_attr}>{run_id}</th>")
|
||||
lines.append("</tr>")
|
||||
for metric_key, label in [
|
||||
("auc", "auc"),
|
||||
("acc", "acc"),
|
||||
("holdout_auc", "holdout_auc"),
|
||||
("holdout_acc", "holdout_acc"),
|
||||
]:
|
||||
lines.append("<tr>")
|
||||
lines.append(f"<th>{label}</th>")
|
||||
for run in runs:
|
||||
metrics = run.get("metrics", {})
|
||||
value = metrics.get(metric_key) if isinstance(metrics, dict) else None
|
||||
lines.append(f"<td>{escape(format_metric(value))}</td>")
|
||||
lines.append("</tr>")
|
||||
lines.append("</thead>")
|
||||
lines.append("<tbody>")
|
||||
for key, value in row_specs:
|
||||
label = f"{key}={format_value(value)}"
|
||||
lines.append("<tr>")
|
||||
lines.append(f"<th class=\"row-label\">{escape(label)}</th>")
|
||||
for run in runs:
|
||||
config = run.get("config", {})
|
||||
current = config.get(key) if isinstance(config, dict) else None
|
||||
cell_class = "on" if current == value else "off"
|
||||
lines.append(f"<td class=\"cell {cell_class}\"></td>")
|
||||
lines.append("</tr>")
|
||||
lines.append("</tbody>")
|
||||
lines.append("</table>")
|
||||
lines.append("</div>")
|
||||
lines.append("</body>")
|
||||
lines.append("</html>")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def truncate(text: str, width: int) -> str:
|
||||
if len(text) <= width:
|
||||
return text
|
||||
if width <= 3:
|
||||
return text[:width]
|
||||
return text[:width - 3] + "..."
|
||||
|
||||
|
||||
def build_text_grid(runs: List[Dict[str, object]],
|
||||
row_specs: List[Tuple[str, object]],
|
||||
filters: List[str],
|
||||
col_width: int,
|
||||
row_width: int,
|
||||
color: bool) -> str:
|
||||
sep = " "
|
||||
lines: List[str] = []
|
||||
if filters:
|
||||
lines.extend(filters)
|
||||
lines.append("")
|
||||
|
||||
def pad(text: str, width: int) -> str:
|
||||
return truncate(text, width).ljust(width)
|
||||
|
||||
def colorize(text: str, enabled: bool) -> str:
|
||||
if not color:
|
||||
return text
|
||||
color_code = "\x1b[32m" if enabled else "\x1b[31m"
|
||||
return f"{color_code}{text}\x1b[0m"
|
||||
|
||||
def row_line(label: str, values: List[str]) -> str:
|
||||
return pad(label, row_width) + sep + sep.join(pad(v, col_width) for v in values)
|
||||
|
||||
run_ids = [str(run.get("run_id", "")) for run in runs]
|
||||
lines.append(row_line("run_id", run_ids))
|
||||
for metric_key, label in [
|
||||
("auc", "auc"),
|
||||
("acc", "acc"),
|
||||
("holdout_auc", "holdout_auc"),
|
||||
("holdout_acc", "holdout_acc"),
|
||||
]:
|
||||
values = []
|
||||
for run in runs:
|
||||
metrics = run.get("metrics", {})
|
||||
value = metrics.get(metric_key) if isinstance(metrics, dict) else None
|
||||
values.append(format_metric(value))
|
||||
lines.append(row_line(label, values))
|
||||
|
||||
divider = "-" * row_width + sep + sep.join("-" * col_width for _ in runs)
|
||||
lines.append(divider)
|
||||
|
||||
for key, value in row_specs:
|
||||
label = f"{key}={format_value(value)}"
|
||||
cells: List[str] = []
|
||||
for run in runs:
|
||||
config = run.get("config", {})
|
||||
current = config.get(key) if isinstance(config, dict) else None
|
||||
enabled = current == value
|
||||
cell = colorize("##", enabled) if enabled else colorize("..", enabled)
|
||||
cells.append(cell)
|
||||
lines.append(row_line(label, cells))
|
||||
|
||||
lines.append("")
|
||||
lines.append("Legend: ##=on ..=off")
|
||||
if color:
|
||||
lines.append("Colors: green=on red=off")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def parse_figsize(value: Optional[str], n_cols: int, n_rows: int) -> Tuple[float, float]:
|
||||
if value:
|
||||
parts = [p.strip() for p in value.split(",") if p.strip()]
|
||||
if len(parts) == 2:
|
||||
try:
|
||||
return float(parts[0]), float(parts[1])
|
||||
except ValueError:
|
||||
pass
|
||||
width = min(40.0, max(8.0, n_cols * 0.3))
|
||||
height = min(40.0, max(6.0, (n_rows + 6) * 0.3))
|
||||
return width, height
|
||||
|
||||
|
||||
def plot_heatmap(runs: List[Dict[str, object]],
|
||||
row_specs: List[Tuple[str, object]],
|
||||
filters: List[str],
|
||||
output_path: Optional[Path],
|
||||
figsize: Tuple[float, float],
|
||||
dpi: int,
|
||||
show: bool) -> None:
|
||||
if not show:
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.colors import ListedColormap
|
||||
|
||||
try:
|
||||
import seaborn as sns
|
||||
except ImportError:
|
||||
sns = None
|
||||
|
||||
metric_labels = ["auc", "acc", "holdout_auc", "holdout_acc"]
|
||||
metric_matrix: List[List[float]] = []
|
||||
for label in metric_labels:
|
||||
row: List[float] = []
|
||||
for run in runs:
|
||||
metrics = run.get("metrics", {})
|
||||
value = metrics.get(label) if isinstance(metrics, dict) else None
|
||||
row.append(float(value) if value is not None else float("nan"))
|
||||
metric_matrix.append(row)
|
||||
|
||||
param_labels = [f"{key}={format_value(value)}" for key, value in row_specs]
|
||||
param_matrix: List[List[int]] = []
|
||||
for key, value in row_specs:
|
||||
row = []
|
||||
for run in runs:
|
||||
config = run.get("config", {})
|
||||
current = config.get(key) if isinstance(config, dict) else None
|
||||
row.append(1 if current == value else 0)
|
||||
param_matrix.append(row)
|
||||
|
||||
fig = plt.figure(figsize=figsize, dpi=dpi)
|
||||
grid_rows = 2 if param_matrix else 1
|
||||
height_ratios = [2, max(2, len(param_matrix) * 0.5)] if param_matrix else [2]
|
||||
gs = fig.add_gridspec(grid_rows, 1, height_ratios=height_ratios, hspace=0.05)
|
||||
|
||||
ax_metrics = fig.add_subplot(gs[0, 0])
|
||||
if sns:
|
||||
sns.heatmap(
|
||||
metric_matrix,
|
||||
ax=ax_metrics,
|
||||
cmap="viridis",
|
||||
annot=True,
|
||||
fmt=".3f",
|
||||
cbar=True,
|
||||
yticklabels=metric_labels,
|
||||
xticklabels=False,
|
||||
)
|
||||
else:
|
||||
im = ax_metrics.imshow(metric_matrix, aspect="auto", cmap="viridis")
|
||||
ax_metrics.set_yticks(range(len(metric_labels)))
|
||||
ax_metrics.set_yticklabels(metric_labels)
|
||||
fig.colorbar(im, ax=ax_metrics, fraction=0.02, pad=0.01)
|
||||
for i, row in enumerate(metric_matrix):
|
||||
for j, value in enumerate(row):
|
||||
if math.isnan(value):
|
||||
continue
|
||||
ax_metrics.text(j, i, f"{value:.3f}", ha="center", va="center", fontsize=7, color="white")
|
||||
ax_metrics.set_ylabel("metrics")
|
||||
|
||||
if param_matrix:
|
||||
ax_params = fig.add_subplot(gs[1, 0], sharex=ax_metrics)
|
||||
cmap = ListedColormap(["#d9534f", "#4caf50"])
|
||||
if sns:
|
||||
sns.heatmap(
|
||||
param_matrix,
|
||||
ax=ax_params,
|
||||
cmap=cmap,
|
||||
cbar=False,
|
||||
yticklabels=param_labels,
|
||||
xticklabels=[run.get("run_id", "") for run in runs],
|
||||
vmin=0,
|
||||
vmax=1,
|
||||
)
|
||||
else:
|
||||
ax_params.imshow(param_matrix, aspect="auto", cmap=cmap, vmin=0, vmax=1)
|
||||
ax_params.set_yticks(range(len(param_labels)))
|
||||
ax_params.set_yticklabels(param_labels)
|
||||
ax_params.set_xticks(range(len(runs)))
|
||||
ax_params.set_xticklabels([run.get("run_id", "") for run in runs], rotation=90)
|
||||
ax_params.set_xlabel("runs")
|
||||
else:
|
||||
ax_metrics.set_xticks(range(len(runs)))
|
||||
ax_metrics.set_xticklabels([run.get("run_id", "") for run in runs], rotation=90)
|
||||
ax_metrics.set_xlabel("runs")
|
||||
|
||||
if filters:
|
||||
fig.suptitle("Grid Search Heatmap\n" + " | ".join(filters), fontsize=10)
|
||||
else:
|
||||
fig.suptitle("Grid Search Heatmap", fontsize=10)
|
||||
|
||||
if output_path is not None:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(output_path, bbox_inches="tight")
|
||||
if show:
|
||||
plt.show()
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description="Build an HTML heatmap grid for grid search runs.")
|
||||
ap.add_argument("--analysis-dir", type=Path, default=Path("analysis_data/grid_search"),
|
||||
help="Directory containing run subdirectories")
|
||||
ap.add_argument("--format", choices=["text", "html", "plot"], default="text",
|
||||
help="Output format (default: text)")
|
||||
ap.add_argument("--task", choices=["binary", "multiclass", "all"], default="all",
|
||||
help="Filter runs by task type (default: all)")
|
||||
ap.add_argument("--sort-by", choices=["auc", "acc", "holdout_auc", "holdout_acc"], default=None,
|
||||
help="Metric to sort columns by (default: none)")
|
||||
ap.add_argument("--asc", action="store_true",
|
||||
help="Sort in ascending order (default: descending)")
|
||||
ap.add_argument("--desc", action="store_true",
|
||||
help="Sort in descending order (default: descending)")
|
||||
ap.add_argument("--top", type=int, default=None,
|
||||
help="Limit to the top N runs after sorting")
|
||||
ap.add_argument("--cluster-rows", dest="cluster_rows", action="store_true",
|
||||
help="Order parameter rows by prevalence in the selected runs (default)")
|
||||
ap.add_argument("--no-cluster-rows", dest="cluster_rows", action="store_false",
|
||||
help="Keep parameter rows sorted alphabetically")
|
||||
ap.set_defaults(cluster_rows=True)
|
||||
ap.add_argument("--output", type=Path, default=None,
|
||||
help="Optional path to write output")
|
||||
ap.add_argument("--params", default=None,
|
||||
help="Comma-separated list of parameter keys to include")
|
||||
ap.add_argument("--exclude", default=None,
|
||||
help="Comma-separated list of parameter keys to exclude")
|
||||
ap.add_argument("--match", default=None,
|
||||
help="Only include run directories whose name contains this substring")
|
||||
ap.add_argument("--shallow", action="store_true",
|
||||
help="Only scan directories directly under analysis-dir")
|
||||
ap.add_argument("--no-progress", action="store_true",
|
||||
help="Disable progress output")
|
||||
ap.add_argument("--col-width", type=int, default=13,
|
||||
help="Column width for text output (default: 13)")
|
||||
ap.add_argument("--row-width", type=int, default=36,
|
||||
help="Row label width for text output (default: 36)")
|
||||
ap.add_argument("--color", action="store_true",
|
||||
help="Use ANSI colors in text output")
|
||||
ap.add_argument("--figsize", default=None,
|
||||
help="Figure size as 'width,height' (inches), for plot output")
|
||||
ap.add_argument("--dpi", type=int, default=140,
|
||||
help="Figure DPI for plot output")
|
||||
ap.add_argument("--show", action="store_true",
|
||||
help="Display plot window (only for format=plot)")
|
||||
args = ap.parse_args()
|
||||
|
||||
root = args.analysis_dir
|
||||
if not root.exists():
|
||||
raise SystemExit(f"Analysis directory not found: {root}")
|
||||
|
||||
exclude_keys = set(DEFAULT_EXCLUDE_KEYS)
|
||||
if args.exclude:
|
||||
for item in args.exclude.split(","):
|
||||
item = item.strip()
|
||||
if item:
|
||||
exclude_keys.add(item)
|
||||
|
||||
runs: List[Dict[str, object]] = []
|
||||
values_by_key: Dict[str, List[object]] = {}
|
||||
missing_summary = 0
|
||||
unknown_task = 0
|
||||
missing_cli = 0
|
||||
|
||||
for run_dir in iter_run_dirs(root, shallow=args.shallow, show_progress=not args.no_progress):
|
||||
if args.match and args.match not in run_dir.name:
|
||||
continue
|
||||
summary = read_summary(run_dir)
|
||||
if summary is None:
|
||||
missing_summary += 1
|
||||
continue
|
||||
task_label = task_from_summary(summary)
|
||||
if args.task != "all":
|
||||
if task_label is None:
|
||||
unknown_task += 1
|
||||
continue
|
||||
if task_label != args.task:
|
||||
continue
|
||||
|
||||
metrics = {
|
||||
"auc": mean_metric(summary, "auc_fused"),
|
||||
"acc": mean_metric(summary, "acc_fused"),
|
||||
"holdout_auc": mean_metric(summary, "holdout_auc_fused"),
|
||||
"holdout_acc": mean_metric(summary, "holdout_acc_fused"),
|
||||
}
|
||||
if any(val is None for val in metrics.values()):
|
||||
continue
|
||||
|
||||
cli_args = read_cli_args(run_dir)
|
||||
if cli_args is None:
|
||||
missing_cli += 1
|
||||
config_source = cli_args if cli_args is not None else summary
|
||||
config = flatten_config(config_source, exclude_keys=exclude_keys)
|
||||
run_id = read_run_id(run_dir, summary)
|
||||
runs.append({
|
||||
"run_id": run_id,
|
||||
"relative_path": str(run_dir.relative_to(root)),
|
||||
"task": task_label,
|
||||
"metrics": metrics,
|
||||
"config": config,
|
||||
})
|
||||
for key, value in config.items():
|
||||
values_by_key.setdefault(key, []).append(value)
|
||||
|
||||
if not runs:
|
||||
print("No matching runs found.")
|
||||
return
|
||||
|
||||
if args.params:
|
||||
param_keys = [p.strip() for p in args.params.split(",") if p.strip()]
|
||||
else:
|
||||
param_keys = []
|
||||
for key, values in values_by_key.items():
|
||||
unique_values = {format_value(v) for v in values}
|
||||
if len(unique_values) > 1:
|
||||
param_keys.append(key)
|
||||
param_keys.sort()
|
||||
|
||||
row_specs: List[Tuple[str, object]] = []
|
||||
for key in param_keys:
|
||||
values = values_by_key.get(key, [])
|
||||
unique_values = []
|
||||
seen = set()
|
||||
for val in values:
|
||||
marker = (type(val), val)
|
||||
if marker in seen:
|
||||
continue
|
||||
seen.add(marker)
|
||||
unique_values.append(val)
|
||||
unique_values.sort(key=sort_value_key)
|
||||
for value in unique_values:
|
||||
row_specs.append((key, value))
|
||||
|
||||
if args.asc and args.desc:
|
||||
raise SystemExit("Choose only one of --asc or --desc.")
|
||||
|
||||
if args.sort_by:
|
||||
def sort_key(item: Dict[str, object]) -> float:
|
||||
metrics = item.get("metrics", {})
|
||||
val = metrics.get(args.sort_by) if isinstance(metrics, dict) else None
|
||||
if val is None:
|
||||
return float("inf") if args.asc else float("-inf")
|
||||
return float(val)
|
||||
|
||||
runs.sort(key=sort_key, reverse=not args.asc)
|
||||
else:
|
||||
runs.sort(key=lambda r: str(r.get("run_id", "")))
|
||||
|
||||
if args.top is not None:
|
||||
runs = runs[:args.top]
|
||||
|
||||
if args.cluster_rows and runs:
|
||||
total = len(runs)
|
||||
counts_by_spec: Dict[Tuple[str, object], int] = {}
|
||||
for key, value in row_specs:
|
||||
counts_by_spec[(key, value)] = 0
|
||||
for run in runs:
|
||||
config = run.get("config", {})
|
||||
if not isinstance(config, dict):
|
||||
continue
|
||||
for key, value in row_specs:
|
||||
if config.get(key) == value:
|
||||
counts_by_spec[(key, value)] += 1
|
||||
|
||||
def row_sort(spec: Tuple[str, object]) -> Tuple[float, str, str]:
|
||||
count = counts_by_spec.get(spec, 0)
|
||||
score = count / total if total else 0.0
|
||||
key, value = spec
|
||||
return (-score, str(key), format_value(value))
|
||||
|
||||
row_specs.sort(key=row_sort)
|
||||
|
||||
filters = []
|
||||
if args.task != "all":
|
||||
filters.append(f"Task filter: {args.task}")
|
||||
if args.match:
|
||||
filters.append(f"Name filter: {args.match}")
|
||||
filters.append(f"Runs: {len(runs)}")
|
||||
filters.append(f"Params: {len(row_specs)}")
|
||||
filters.append(f"Row clustering: {'on' if args.cluster_rows else 'off'}")
|
||||
if missing_summary or unknown_task:
|
||||
filters.append(f"Skipped: {missing_summary} missing summary, {unknown_task} unknown task")
|
||||
if missing_cli:
|
||||
filters.append(f"Missing cli_args: {missing_cli}")
|
||||
|
||||
title = "Grid Search Heatmap"
|
||||
if args.format == "html":
|
||||
html = build_html(runs, row_specs, title=title, filters=filters)
|
||||
output_path = args.output or Path("analysis_data/grid_search_heatmap.html")
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(html)
|
||||
print(f"Wrote {output_path}")
|
||||
elif args.format == "plot":
|
||||
output_path = args.output
|
||||
if output_path is None and not args.show:
|
||||
output_path = Path("analysis_data/grid_search_heatmap.png")
|
||||
figsize = parse_figsize(args.figsize, n_cols=len(runs), n_rows=len(row_specs))
|
||||
plot_heatmap(
|
||||
runs,
|
||||
row_specs,
|
||||
filters=filters,
|
||||
output_path=output_path,
|
||||
figsize=figsize,
|
||||
dpi=args.dpi,
|
||||
show=args.show,
|
||||
)
|
||||
if output_path is not None:
|
||||
print(f"Wrote {output_path}")
|
||||
else:
|
||||
text = build_text_grid(
|
||||
runs,
|
||||
row_specs,
|
||||
filters=filters,
|
||||
col_width=max(4, args.col_width),
|
||||
row_width=max(12, args.row_width),
|
||||
color=args.color,
|
||||
)
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(text)
|
||||
print(f"Wrote {args.output}")
|
||||
else:
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,572 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Plot holdout ROC curves for a specific grid search run."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from sklearn.metrics import auc, roc_curve
|
||||
|
||||
|
||||
# ---------------------------
|
||||
# Config (edit in IDE)
|
||||
# ---------------------------
|
||||
RUN_ID = "20251129-0312"
|
||||
ANALYSIS_ROOT = Path("analysis_data/grid_search")
|
||||
HEADS = ["fused", "image", "metadata"]
|
||||
OUTPUT_SUBDIR = Path("plots/holdout_rocs")
|
||||
BEST_OUTPUT_SUBDIR = Path("plots/best_rocs")
|
||||
POSITIVE_CLASS = 1
|
||||
DEBUG = True
|
||||
USE_JSON_ROC = True
|
||||
USE_HOLDOUT_PROBS = True
|
||||
ALLOW_FALLBACK_TO_VALIDATION = False
|
||||
PLOT_VALIDATION_FROM_HOLDOUT_EPOCH = True
|
||||
PLOT_BEST_EPOCH = True
|
||||
PLOT_HOLDOUT_FROM_BEST_EPOCH = True
|
||||
PLOT_ALL_CLASSES = True
|
||||
FORCE_PROBS_FOR_BEST_BINARY = True
|
||||
FORCE_PROBS_FOR_HOLDOUT_BINARY = False
|
||||
|
||||
|
||||
HEAD_FILE_KEYS = {
|
||||
"fused": "fused",
|
||||
"image": "img",
|
||||
"metadata": "md",
|
||||
}
|
||||
|
||||
JSON_DIR_NAMES = [
|
||||
"roc_curves_holdout_best",
|
||||
"roc_curves",
|
||||
]
|
||||
|
||||
|
||||
def _epoch_from_name(path: Path) -> int:
|
||||
m = re.search(r"epoch(\d+)", path.name)
|
||||
return int(m.group(1)) if m else -1
|
||||
|
||||
|
||||
def _load_json(path: Path) -> Dict:
|
||||
try:
|
||||
return json.loads(path.read_text())
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _infer_run_info(run_dir: Path) -> Tuple[str | None, int | None, List[str] | None]:
|
||||
cli = _load_json(run_dir / "cli_args.json")
|
||||
summary = _load_json(run_dir / "summary.json")
|
||||
payloads = [cli, summary]
|
||||
eval_mode = None
|
||||
num_classes = None
|
||||
class_names = None
|
||||
for payload in payloads:
|
||||
if not payload:
|
||||
continue
|
||||
if eval_mode is None:
|
||||
em = payload.get("eval_mode")
|
||||
if isinstance(em, str):
|
||||
eval_mode = em.strip().lower()
|
||||
if num_classes is None:
|
||||
nc = payload.get("num_classes")
|
||||
if isinstance(nc, (int, float)):
|
||||
num_classes = int(nc)
|
||||
if class_names is None:
|
||||
cn = payload.get("class_names")
|
||||
if isinstance(cn, list) and cn:
|
||||
class_names = [str(x) for x in cn]
|
||||
if num_classes is None and eval_mode:
|
||||
num_classes = 2 if eval_mode == "binary" else 3
|
||||
return eval_mode, num_classes, class_names
|
||||
|
||||
|
||||
def _collect_holdout_json_files(run_dir: Path, head: str) -> Dict[int, Path]:
|
||||
fold_files: Dict[int, Path] = {}
|
||||
# Fold-scoped folders
|
||||
for folder_name in JSON_DIR_NAMES:
|
||||
for fold_dir in run_dir.glob(f"fold*_{folder_name}"):
|
||||
fold_match = re.search(r"fold(\d+)_", fold_dir.name)
|
||||
if not fold_match:
|
||||
continue
|
||||
fold_idx = int(fold_match.group(1))
|
||||
candidates = list(fold_dir.glob(f"epoch*_holdout_{head}.json"))
|
||||
if not candidates:
|
||||
candidates = list(fold_dir.glob(f"epoch*_{head}.json"))
|
||||
if candidates:
|
||||
candidates.sort(key=_epoch_from_name)
|
||||
fold_files[fold_idx] = candidates[-1]
|
||||
if fold_files:
|
||||
return fold_files
|
||||
# Fallback: unscoped roc_curves in run_dir (single-fold or in-progress)
|
||||
for folder_name in JSON_DIR_NAMES:
|
||||
base_dir = run_dir / folder_name
|
||||
if not base_dir.exists():
|
||||
continue
|
||||
candidates = list(base_dir.glob(f"epoch*_holdout_{head}.json"))
|
||||
if not candidates:
|
||||
candidates = list(base_dir.glob(f"epoch*_{head}.json"))
|
||||
if candidates:
|
||||
candidates.sort(key=_epoch_from_name)
|
||||
fold_files[0] = candidates[-1]
|
||||
break
|
||||
return fold_files
|
||||
|
||||
|
||||
def _collect_validation_json_files(
|
||||
holdout_files: Dict[int, Path], head: str
|
||||
) -> Dict[int, Path]:
|
||||
validation_files: Dict[int, Path] = {}
|
||||
for fold_idx, holdout_path in holdout_files.items():
|
||||
epoch = _epoch_from_name(holdout_path)
|
||||
if epoch < 0:
|
||||
continue
|
||||
candidate = holdout_path.parent / f"epoch{epoch}_{head}.json"
|
||||
if candidate.exists():
|
||||
validation_files[fold_idx] = candidate
|
||||
continue
|
||||
# Fallback: try the same epoch under roc_curves (if holdout_best folder omitted it).
|
||||
for folder_name in JSON_DIR_NAMES:
|
||||
alt_dir = holdout_path.parent.parent / f"fold{fold_idx}_{folder_name}"
|
||||
alt_candidate = alt_dir / f"epoch{epoch}_{head}.json"
|
||||
if alt_candidate.exists():
|
||||
validation_files[fold_idx] = alt_candidate
|
||||
break
|
||||
return validation_files
|
||||
|
||||
|
||||
def _collect_holdout_from_validation_files(
|
||||
validation_files: Dict[int, Path], head: str
|
||||
) -> Dict[int, Path]:
|
||||
holdout_files: Dict[int, Path] = {}
|
||||
for fold_idx, val_path in validation_files.items():
|
||||
epoch = _epoch_from_name(val_path)
|
||||
if epoch < 0:
|
||||
continue
|
||||
candidate = val_path.parent / f"epoch{epoch}_holdout_{head}.json"
|
||||
if candidate.exists():
|
||||
holdout_files[fold_idx] = candidate
|
||||
continue
|
||||
for folder_name in ("roc_curves", "roc_curves_holdout_best"):
|
||||
alt_dir = val_path.parent.parent / f"fold{fold_idx}_{folder_name}"
|
||||
alt_candidate = alt_dir / f"epoch{epoch}_holdout_{head}.json"
|
||||
if alt_candidate.exists():
|
||||
holdout_files[fold_idx] = alt_candidate
|
||||
break
|
||||
return holdout_files
|
||||
|
||||
|
||||
def _collect_best_json_files(run_dir: Path, head: str) -> Dict[int, Path]:
|
||||
fold_files: Dict[int, Path] = {}
|
||||
for fold_dir in run_dir.glob("fold*_roc_curves_best"):
|
||||
fold_match = re.search(r"fold(\d+)_", fold_dir.name)
|
||||
if not fold_match:
|
||||
continue
|
||||
fold_idx = int(fold_match.group(1))
|
||||
candidates = list(fold_dir.glob(f"epoch*_{head}.json"))
|
||||
if candidates:
|
||||
candidates.sort(key=_epoch_from_name)
|
||||
fold_files[fold_idx] = candidates[-1]
|
||||
return fold_files
|
||||
|
||||
|
||||
def _extract_curves(data: Dict) -> Dict[str, Tuple[List[float], List[float], float]]:
|
||||
curves: Dict[str, Tuple[List[float], List[float], float]] = {}
|
||||
per_class = data.get("per_class") if isinstance(data, dict) else None
|
||||
if not isinstance(per_class, dict):
|
||||
return curves
|
||||
for cls, entry in per_class.items():
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
fpr = entry.get("fpr")
|
||||
tpr = entry.get("tpr")
|
||||
auc_val = entry.get("auc")
|
||||
if not isinstance(fpr, list) or not isinstance(tpr, list):
|
||||
continue
|
||||
try:
|
||||
auc_f = float(auc_val) if auc_val is not None else float("nan")
|
||||
except Exception:
|
||||
auc_f = float("nan")
|
||||
curves[str(cls)] = (fpr, tpr, auc_f)
|
||||
return curves
|
||||
|
||||
|
||||
def _derive_positive_from_class0(
|
||||
curves_by_class: Dict[str, List[Tuple[int, List[float], List[float], float]]],
|
||||
positive_class: int,
|
||||
) -> None:
|
||||
zero_key = "0"
|
||||
if zero_key not in curves_by_class:
|
||||
return
|
||||
derived = []
|
||||
for fold_idx, fpr0, tpr0, auc0 in curves_by_class.get(zero_key, []):
|
||||
# The JSON for binary currently stores class-1 labels with class-0 scores,
|
||||
# so invert the curve to recover the true class-1 ROC.
|
||||
fpr1 = [1.0 - float(x) for x in fpr0]
|
||||
tpr1 = [1.0 - float(x) for x in tpr0]
|
||||
# Ensure increasing FPR for plotting.
|
||||
if len(fpr1) > 1 and fpr1[0] > fpr1[-1]:
|
||||
fpr1 = list(reversed(fpr1))
|
||||
tpr1 = list(reversed(tpr1))
|
||||
auc1 = 1.0 - auc0 if auc0 == auc0 else auc0
|
||||
derived.append((fold_idx, fpr1, tpr1, auc1))
|
||||
curves_by_class[str(positive_class)] = derived
|
||||
|
||||
|
||||
def _needs_positive_derivation(
|
||||
curves_by_class: Dict[str, List[Tuple[int, List[float], List[float], float]]],
|
||||
positive_class: int,
|
||||
) -> bool:
|
||||
curves = curves_by_class.get(str(positive_class))
|
||||
if not curves:
|
||||
return True
|
||||
for _, fpr, tpr, auc_val in curves:
|
||||
if auc_val == auc_val and len(fpr) > 2 and len(tpr) > 2:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _collect_prob_files(run_dir: Path, suffix: str) -> Dict[int, Dict[str, Path]]:
|
||||
files: Dict[int, Dict[str, Path]] = {}
|
||||
for y_file in run_dir.glob(f"fold*_y_true{suffix}.npy"):
|
||||
fold_str = y_file.stem.split("_")[0].replace("fold", "")
|
||||
try:
|
||||
fold_idx = int(fold_str)
|
||||
except ValueError:
|
||||
continue
|
||||
files.setdefault(fold_idx, {})["y_true"] = y_file
|
||||
for head, key in HEAD_FILE_KEYS.items():
|
||||
for p_file in run_dir.glob(f"fold*_probs_{key}{suffix}.npy"):
|
||||
fold_str = p_file.stem.split("_")[0].replace("fold", "")
|
||||
try:
|
||||
fold_idx = int(fold_str)
|
||||
except ValueError:
|
||||
continue
|
||||
files.setdefault(fold_idx, {})[head] = p_file
|
||||
return files
|
||||
|
||||
|
||||
def _load_array(path: Path) -> np.ndarray | None:
|
||||
try:
|
||||
return np.load(path)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _compute_binary_curve(
|
||||
y_true: np.ndarray, probs: np.ndarray, positive_class: int
|
||||
) -> Tuple[List[float], List[float], float] | None:
|
||||
if probs.ndim == 1:
|
||||
scores = probs
|
||||
elif probs.ndim == 2 and probs.shape[1] > positive_class:
|
||||
scores = probs[:, positive_class]
|
||||
else:
|
||||
return None
|
||||
y_bin = (y_true == positive_class).astype(int)
|
||||
if y_bin.sum() == 0 or y_bin.sum() == len(y_bin):
|
||||
return None
|
||||
fpr, tpr, _ = roc_curve(y_bin, scores)
|
||||
auc_val = float(auc(fpr, tpr))
|
||||
return fpr.tolist(), tpr.tolist(), auc_val
|
||||
|
||||
|
||||
def _compute_multiclass_curves(
|
||||
y_true: np.ndarray, probs: np.ndarray
|
||||
) -> Dict[str, Tuple[List[float], List[float], float]]:
|
||||
curves: Dict[str, Tuple[List[float], List[float], float]] = {}
|
||||
if probs.ndim != 2:
|
||||
return curves
|
||||
num_classes = probs.shape[1]
|
||||
for cls in range(num_classes):
|
||||
y_bin = (y_true == cls).astype(int)
|
||||
if y_bin.sum() == 0 or y_bin.sum() == len(y_bin):
|
||||
continue
|
||||
fpr, tpr, _ = roc_curve(y_bin, probs[:, cls])
|
||||
curves[str(cls)] = (fpr.tolist(), tpr.tolist(), float(auc(fpr, tpr)))
|
||||
return curves
|
||||
|
||||
|
||||
def _plot_overlays(
|
||||
curves_by_fold: List[Tuple[int, List[float], List[float], float]],
|
||||
title: str,
|
||||
out_path: Path,
|
||||
) -> None:
|
||||
fig, ax = plt.subplots(figsize=(6, 5))
|
||||
for fold_idx, fpr, tpr, auc_val in curves_by_fold:
|
||||
label = (
|
||||
f"fold{fold_idx} AUC={auc_val:.3f}"
|
||||
if auc_val == auc_val
|
||||
else f"fold{fold_idx}"
|
||||
)
|
||||
ax.plot(fpr, tpr, lw=1.4, label=label)
|
||||
ax.plot([0, 1], [0, 1], "k--", lw=1)
|
||||
ax.set_xlabel("False Positive Rate")
|
||||
ax.set_ylabel("True Positive Rate")
|
||||
ax.set_title(title)
|
||||
ax.legend(loc="lower right", fontsize="small")
|
||||
ax.grid(True, alpha=0.3, linestyle="--")
|
||||
fig.tight_layout()
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out_path, dpi=170)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
run_dir = ANALYSIS_ROOT / RUN_ID
|
||||
if not run_dir.exists():
|
||||
raise SystemExit(f"Run not found: {run_dir}")
|
||||
eval_mode, num_classes, class_names = _infer_run_info(run_dir)
|
||||
is_binary = eval_mode == "binary" or num_classes == 2
|
||||
|
||||
def _plot_set(
|
||||
label: str,
|
||||
head: str,
|
||||
json_files: Dict[int, Path],
|
||||
out_dir: Path,
|
||||
paired_files: Dict[int, Path] | None,
|
||||
paired_suffix: str,
|
||||
class_names: List[str] | None,
|
||||
) -> None:
|
||||
if not json_files:
|
||||
return
|
||||
curves_by_class: Dict[
|
||||
str, List[Tuple[int, List[float], List[float], float]]
|
||||
] = {}
|
||||
paired_curves_by_class: Dict[
|
||||
str, List[Tuple[int, List[float], List[float], float]]
|
||||
] = {}
|
||||
for fold_idx, path in sorted(json_files.items()):
|
||||
if DEBUG:
|
||||
print(f"[debug] {label} head={head} fold={fold_idx} json={path}")
|
||||
data = _load_json(path)
|
||||
curves = _extract_curves(data)
|
||||
for cls, (fpr, tpr, auc_val) in curves.items():
|
||||
curves_by_class.setdefault(cls, []).append(
|
||||
(fold_idx, fpr, tpr, auc_val)
|
||||
)
|
||||
if paired_files:
|
||||
p_path = paired_files.get(fold_idx)
|
||||
if p_path is not None:
|
||||
if DEBUG:
|
||||
print(
|
||||
f"[debug] {label} head={head} fold={fold_idx} paired_json={p_path}"
|
||||
)
|
||||
p_data = _load_json(p_path)
|
||||
p_curves = _extract_curves(p_data)
|
||||
for cls, (fpr, tpr, auc_val) in p_curves.items():
|
||||
paired_curves_by_class.setdefault(cls, []).append(
|
||||
(fold_idx, fpr, tpr, auc_val)
|
||||
)
|
||||
if _needs_positive_derivation(curves_by_class, POSITIVE_CLASS):
|
||||
_derive_positive_from_class0(curves_by_class, POSITIVE_CLASS)
|
||||
if paired_curves_by_class and _needs_positive_derivation(
|
||||
paired_curves_by_class, POSITIVE_CLASS
|
||||
):
|
||||
_derive_positive_from_class0(paired_curves_by_class, POSITIVE_CLASS)
|
||||
|
||||
if PLOT_ALL_CLASSES:
|
||||
classes = list(curves_by_class.keys())
|
||||
else:
|
||||
classes = (
|
||||
[str(POSITIVE_CLASS)]
|
||||
if str(POSITIVE_CLASS) in curves_by_class
|
||||
else list(curves_by_class.keys())
|
||||
)
|
||||
if not classes:
|
||||
return
|
||||
|
||||
for cls in classes:
|
||||
fold_curves = curves_by_class.get(cls, [])
|
||||
if not fold_curves:
|
||||
continue
|
||||
class_label = cls
|
||||
if class_names is not None:
|
||||
try:
|
||||
idx = int(cls)
|
||||
if 0 <= idx < len(class_names):
|
||||
class_label = f"{cls} ({class_names[idx]})"
|
||||
except Exception:
|
||||
pass
|
||||
title = f"{RUN_ID} {label} ROC — head={head} class={class_label}"
|
||||
out_path = out_dir / f"{label}_{head}_class{cls}.png"
|
||||
_plot_overlays(fold_curves, title, out_path)
|
||||
print(f"[ok] {out_path}")
|
||||
|
||||
if paired_curves_by_class:
|
||||
p_curves = paired_curves_by_class.get(cls, [])
|
||||
if p_curves:
|
||||
p_title = f"{RUN_ID} {label} {paired_suffix} ROC — head={head} class={class_label}"
|
||||
p_path = out_dir / f"{label}_{head}_class{cls}_{paired_suffix}.png"
|
||||
_plot_overlays(p_curves, p_title, p_path)
|
||||
print(f"[ok] {p_path}")
|
||||
|
||||
def _plot_from_probs(
|
||||
label: str,
|
||||
head: str,
|
||||
out_dir: Path,
|
||||
suffix: str,
|
||||
class_names: List[str] | None,
|
||||
) -> None:
|
||||
curves_by_class: Dict[
|
||||
str, List[Tuple[int, List[float], List[float], float]]
|
||||
] = {}
|
||||
files = _collect_prob_files(run_dir, suffix)
|
||||
if not files and suffix and ALLOW_FALLBACK_TO_VALIDATION:
|
||||
files = _collect_prob_files(run_dir, "")
|
||||
if files:
|
||||
print(
|
||||
"[warn] Holdout probability dumps not found; using validation probabilities instead."
|
||||
)
|
||||
if not files:
|
||||
return
|
||||
for fold_idx in sorted(files.keys()):
|
||||
fold_files = files[fold_idx]
|
||||
y_path = fold_files.get("y_true")
|
||||
p_path = fold_files.get(head)
|
||||
if y_path is None or p_path is None:
|
||||
continue
|
||||
y_true = _load_array(y_path)
|
||||
probs = _load_array(p_path)
|
||||
if y_true is None or probs is None:
|
||||
continue
|
||||
curves = _compute_multiclass_curves(y_true, probs)
|
||||
for cls, payload in curves.items():
|
||||
curves_by_class.setdefault(cls, []).append((fold_idx, *payload))
|
||||
if not curves_by_class:
|
||||
return
|
||||
classes = list(curves_by_class.keys())
|
||||
for cls in classes:
|
||||
fold_curves = curves_by_class.get(cls, [])
|
||||
if not fold_curves:
|
||||
continue
|
||||
class_label = cls
|
||||
if class_names is not None:
|
||||
try:
|
||||
idx = int(cls)
|
||||
if 0 <= idx < len(class_names):
|
||||
class_label = f"{cls} ({class_names[idx]})"
|
||||
except Exception:
|
||||
pass
|
||||
title = f"{RUN_ID} {label} ROC — head={head} class={class_label}"
|
||||
out_path = out_dir / f"{label}_{head}_class{cls}.png"
|
||||
_plot_overlays(fold_curves, title, out_path)
|
||||
print(f"[ok] {out_path}")
|
||||
|
||||
any_holdout_json = False
|
||||
if USE_JSON_ROC:
|
||||
out_dir = run_dir / OUTPUT_SUBDIR
|
||||
for head in HEADS:
|
||||
files = _collect_holdout_json_files(run_dir, head)
|
||||
if files:
|
||||
any_holdout_json = True
|
||||
paired = (
|
||||
_collect_validation_json_files(files, head)
|
||||
if PLOT_VALIDATION_FROM_HOLDOUT_EPOCH
|
||||
else None
|
||||
)
|
||||
if is_binary and FORCE_PROBS_FOR_HOLDOUT_BINARY:
|
||||
_plot_from_probs("holdout", head, out_dir, "_holdout", class_names)
|
||||
else:
|
||||
_plot_set(
|
||||
"holdout",
|
||||
head,
|
||||
files,
|
||||
out_dir,
|
||||
paired,
|
||||
"validation",
|
||||
class_names,
|
||||
)
|
||||
if not any_holdout_json and DEBUG:
|
||||
print("[debug] no JSON ROC files found; falling back to probs")
|
||||
|
||||
if PLOT_BEST_EPOCH:
|
||||
best_out_dir = run_dir / BEST_OUTPUT_SUBDIR
|
||||
for head in HEADS:
|
||||
if is_binary and FORCE_PROBS_FOR_BEST_BINARY:
|
||||
_plot_from_probs("best", head, best_out_dir, "", class_names)
|
||||
continue
|
||||
best_files = _collect_best_json_files(run_dir, head)
|
||||
if best_files:
|
||||
paired = (
|
||||
_collect_holdout_from_validation_files(best_files, head)
|
||||
if PLOT_HOLDOUT_FROM_BEST_EPOCH
|
||||
else None
|
||||
)
|
||||
_plot_set(
|
||||
"best",
|
||||
head,
|
||||
best_files,
|
||||
best_out_dir,
|
||||
paired,
|
||||
"holdout",
|
||||
class_names,
|
||||
)
|
||||
|
||||
# Fallback to probs for holdout plots if JSON wasn't found.
|
||||
if USE_JSON_ROC and any_holdout_json:
|
||||
return
|
||||
|
||||
out_dir = run_dir / OUTPUT_SUBDIR
|
||||
for head in HEADS:
|
||||
curves_by_class: Dict[
|
||||
str, List[Tuple[int, List[float], List[float], float]]
|
||||
] = {}
|
||||
suffix = "_holdout" if USE_HOLDOUT_PROBS else ""
|
||||
files = _collect_prob_files(run_dir, suffix)
|
||||
if not files and USE_HOLDOUT_PROBS and ALLOW_FALLBACK_TO_VALIDATION:
|
||||
suffix = ""
|
||||
files = _collect_prob_files(run_dir, suffix)
|
||||
if files:
|
||||
print(
|
||||
"[warn] Holdout probability dumps not found; using validation probabilities instead."
|
||||
)
|
||||
if not files:
|
||||
raise SystemExit(
|
||||
"No saved probability dumps found. If you want holdout ROC curves, "
|
||||
"run scripts/rebuild_run_best_plots.py with --use-holdout --overwrite "
|
||||
"to generate fold*_y_true_holdout.npy and fold*_probs_*_holdout.npy files."
|
||||
)
|
||||
for fold_idx in sorted(files.keys()):
|
||||
fold_files = files[fold_idx]
|
||||
y_path = fold_files.get("y_true")
|
||||
p_path = fold_files.get(head)
|
||||
if y_path is None or p_path is None:
|
||||
continue
|
||||
y_true = _load_array(y_path)
|
||||
probs = _load_array(p_path)
|
||||
if y_true is None or probs is None:
|
||||
continue
|
||||
curves = _compute_multiclass_curves(y_true, probs)
|
||||
for cls, payload in curves.items():
|
||||
if payload is None:
|
||||
continue
|
||||
fpr, tpr, auc_val = payload
|
||||
curves_by_class.setdefault(cls, []).append(
|
||||
(fold_idx, fpr, tpr, auc_val)
|
||||
)
|
||||
if not curves_by_class:
|
||||
continue
|
||||
classes = sorted(curves_by_class.keys(), key=lambda x: (float(x), str(x)))
|
||||
for cls in classes:
|
||||
fold_curves = curves_by_class.get(cls, [])
|
||||
if not fold_curves:
|
||||
continue
|
||||
class_label = cls
|
||||
if class_names is not None:
|
||||
try:
|
||||
idx = int(cls)
|
||||
if 0 <= idx < len(class_names):
|
||||
class_label = f"{cls} ({class_names[idx]})"
|
||||
except Exception:
|
||||
pass
|
||||
title = f"{RUN_ID} holdout ROC — head={head} class={class_label}"
|
||||
out_path = out_dir / f"holdout_{head}_class{cls}.png"
|
||||
_plot_overlays(fold_curves, title, out_path)
|
||||
print(f"[ok] {out_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,514 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Analyze correlations between grid search parameters and performance metrics.
|
||||
|
||||
Example:
|
||||
python scripts/grid_search_analytics/param_perf_correlations.py \
|
||||
--analysis-dir analysis_data/grid_search \
|
||||
--task binary \
|
||||
--metric holdout_auc \
|
||||
--top 30
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
DEFAULT_EXCLUDE_KEYS = {
|
||||
"run_id",
|
||||
"fold_metrics",
|
||||
"best_metric",
|
||||
"best_metric_mode",
|
||||
"best_metric_mean",
|
||||
"best_metric_std",
|
||||
"eval_mode",
|
||||
"n_splits",
|
||||
"num_classes",
|
||||
}
|
||||
|
||||
METRIC_MAP = {
|
||||
"auc": "auc_fused",
|
||||
"acc": "acc_fused",
|
||||
"holdout_auc": "holdout_auc_fused",
|
||||
"holdout_acc": "holdout_acc_fused",
|
||||
}
|
||||
|
||||
|
||||
def to_float(value: Optional[object]) -> Optional[float]:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
num = float(value)
|
||||
if math.isnan(num):
|
||||
return None
|
||||
return num
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
value = value.strip()
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
num = float(value)
|
||||
except ValueError:
|
||||
return None
|
||||
if math.isnan(num):
|
||||
return None
|
||||
return num
|
||||
|
||||
|
||||
def mean(values: List[float]) -> Optional[float]:
|
||||
return (sum(values) / len(values)) if values else None
|
||||
|
||||
|
||||
def read_json(path: Path) -> Optional[Dict[str, object]]:
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
data = json.loads(path.read_text())
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
def read_summary(run_dir: Path) -> Optional[Dict[str, object]]:
|
||||
return read_json(run_dir / "summary.json")
|
||||
|
||||
|
||||
def read_cli_args(run_dir: Path) -> Optional[Dict[str, object]]:
|
||||
return read_json(run_dir / "cli_args.json")
|
||||
|
||||
|
||||
def read_run_id(run_dir: Path, summary: Optional[Dict[str, object]]) -> str:
|
||||
if summary:
|
||||
rid = summary.get("run_id")
|
||||
if isinstance(rid, str) and rid:
|
||||
return rid
|
||||
return run_dir.name
|
||||
|
||||
|
||||
def task_from_summary(summary: Optional[Dict[str, object]]) -> Optional[str]:
|
||||
if not summary:
|
||||
return None
|
||||
eval_mode = summary.get("eval_mode")
|
||||
if isinstance(eval_mode, str):
|
||||
mode = eval_mode.strip().lower()
|
||||
if mode == "binary":
|
||||
return "binary"
|
||||
if mode in {"multiclass", "multi", "multi-class"}:
|
||||
return "multiclass"
|
||||
num_classes = summary.get("num_classes")
|
||||
if isinstance(num_classes, (int, float)):
|
||||
return "binary" if int(num_classes) <= 2 else "multiclass"
|
||||
return None
|
||||
|
||||
|
||||
def metric_from_stats(stats: Dict[str, object], metric: str) -> Optional[float]:
|
||||
if metric.startswith("holdout_") and stats.get("holdout_best_monitor") == metric:
|
||||
best_val = to_float(stats.get("holdout_best_so_far"))
|
||||
if best_val is not None:
|
||||
return best_val
|
||||
return to_float(stats.get(metric))
|
||||
|
||||
|
||||
def mean_metric(summary: Dict[str, object], metric: str) -> Optional[float]:
|
||||
folds = summary.get("fold_metrics") or []
|
||||
if not isinstance(folds, list) or not folds:
|
||||
return None
|
||||
values = []
|
||||
for fold in folds:
|
||||
stats = fold.get("stats") if isinstance(fold, dict) else None
|
||||
if not isinstance(stats, dict):
|
||||
return None
|
||||
val = metric_from_stats(stats, metric)
|
||||
if val is None:
|
||||
return None
|
||||
values.append(val)
|
||||
return mean(values)
|
||||
|
||||
|
||||
def flatten_config(data: Dict[str, object],
|
||||
prefix: str = "",
|
||||
exclude_keys: Optional[Iterable[str]] = None) -> Dict[str, object]:
|
||||
out: Dict[str, object] = {}
|
||||
excludes = set(exclude_keys or [])
|
||||
for key, value in data.items():
|
||||
if key in excludes or key.startswith("best_"):
|
||||
continue
|
||||
full_key = f"{prefix}{key}" if not prefix else f"{prefix}.{key}"
|
||||
if isinstance(value, dict):
|
||||
out.update(flatten_config(value, full_key, exclude_keys=excludes))
|
||||
continue
|
||||
if isinstance(value, list):
|
||||
continue
|
||||
out[full_key] = value
|
||||
return out
|
||||
|
||||
|
||||
def rankdata(values: List[float]) -> List[float]:
|
||||
order = sorted(range(len(values)), key=lambda i: values[i])
|
||||
ranks = [0.0] * len(values)
|
||||
i = 0
|
||||
while i < len(values):
|
||||
j = i
|
||||
while j + 1 < len(values) and values[order[j + 1]] == values[order[i]]:
|
||||
j += 1
|
||||
avg_rank = (i + j) / 2.0 + 1.0
|
||||
for k in range(i, j + 1):
|
||||
ranks[order[k]] = avg_rank
|
||||
i = j + 1
|
||||
return ranks
|
||||
|
||||
|
||||
def pearson(x: List[float], y: List[float]) -> Optional[float]:
|
||||
if len(x) != len(y) or len(x) < 2:
|
||||
return None
|
||||
mean_x = sum(x) / len(x)
|
||||
mean_y = sum(y) / len(y)
|
||||
num = sum((xi - mean_x) * (yi - mean_y) for xi, yi in zip(x, y))
|
||||
den_x = sum((xi - mean_x) ** 2 for xi in x)
|
||||
den_y = sum((yi - mean_y) ** 2 for yi in y)
|
||||
if den_x <= 0 or den_y <= 0:
|
||||
return None
|
||||
return num / math.sqrt(den_x * den_y)
|
||||
|
||||
|
||||
def spearman(x: List[float], y: List[float]) -> Optional[float]:
|
||||
rx = rankdata(x)
|
||||
ry = rankdata(y)
|
||||
return pearson(rx, ry)
|
||||
|
||||
|
||||
def correlation_ratio(categories: List[object], values: List[float]) -> Optional[float]:
|
||||
if len(categories) != len(values) or len(values) < 2:
|
||||
return None
|
||||
overall = mean(values)
|
||||
if overall is None:
|
||||
return None
|
||||
total = sum((v - overall) ** 2 for v in values)
|
||||
if total <= 0:
|
||||
return None
|
||||
sums: Dict[object, List[float]] = {}
|
||||
for cat, val in zip(categories, values):
|
||||
sums.setdefault(cat, []).append(val)
|
||||
between = 0.0
|
||||
for vals in sums.values():
|
||||
avg = mean(vals)
|
||||
if avg is None:
|
||||
continue
|
||||
between += len(vals) * (avg - overall) ** 2
|
||||
return math.sqrt(between / total)
|
||||
|
||||
|
||||
def format_value(value: object) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if isinstance(value, int):
|
||||
return str(value)
|
||||
if isinstance(value, float):
|
||||
return f"{value:.6g}"
|
||||
return str(value)
|
||||
|
||||
|
||||
def format_metric(value: Optional[float]) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
return f"{value:.4f}"
|
||||
|
||||
|
||||
def format_table(rows: List[Dict[str, object]], columns: List[str]) -> str:
|
||||
col_widths = {
|
||||
col: max(len(col), max((len(str(row.get(col, ""))) for row in rows), default=0))
|
||||
for col in columns
|
||||
}
|
||||
header = " | ".join(col.ljust(col_widths[col]) for col in columns)
|
||||
divider = "-+-".join("-" * col_widths[col] for col in columns)
|
||||
body = [
|
||||
" | ".join(str(row.get(col, "")).ljust(col_widths[col]) for col in columns)
|
||||
for row in rows
|
||||
]
|
||||
return "\n".join([header, divider, *body])
|
||||
|
||||
|
||||
def render_progress(current: int, total: Optional[int], matched: int) -> str:
|
||||
if total:
|
||||
width = 30
|
||||
filled = int(width * current / total)
|
||||
bar = "#" * filled + "-" * (width - filled)
|
||||
return f"[{bar}] {current}/{total} matched {matched}"
|
||||
return f"Scanned {current} dirs, matched {matched}"
|
||||
|
||||
|
||||
def iter_run_dirs(root: Path, shallow: bool, show_progress: bool) -> Iterable[Path]:
|
||||
if shallow:
|
||||
entries = [entry for entry in root.iterdir() if entry.is_dir()]
|
||||
entries.sort(key=lambda p: p.name)
|
||||
total = len(entries)
|
||||
matched = 0
|
||||
last_update = 0.0
|
||||
for idx, entry in enumerate(entries, start=1):
|
||||
if show_progress:
|
||||
now = time.monotonic()
|
||||
if now - last_update >= 0.1 or idx == total:
|
||||
msg = render_progress(idx, total, matched)
|
||||
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
|
||||
last_update = now
|
||||
if (entry / "summary.json").is_file():
|
||||
matched += 1
|
||||
yield entry
|
||||
if show_progress:
|
||||
print(file=sys.stderr)
|
||||
return
|
||||
|
||||
matched = 0
|
||||
scanned = 0
|
||||
last_update = 0.0
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
scanned += 1
|
||||
if show_progress:
|
||||
now = time.monotonic()
|
||||
if now - last_update >= 0.2:
|
||||
msg = render_progress(scanned, None, matched)
|
||||
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
|
||||
last_update = now
|
||||
if "summary.json" in filenames:
|
||||
matched += 1
|
||||
yield Path(dirpath)
|
||||
if show_progress:
|
||||
msg = render_progress(scanned, None, matched)
|
||||
print(f"\rScanning {msg}", end="", file=sys.stderr, flush=True)
|
||||
print(file=sys.stderr)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description="Correlate grid search parameters with performance.")
|
||||
ap.add_argument("--analysis-dir", type=Path, default=Path("analysis_data/grid_search"),
|
||||
help="Directory containing run subdirectories")
|
||||
ap.add_argument("--task", choices=["binary", "multiclass", "all"], default="all",
|
||||
help="Filter runs by task type (default: all)")
|
||||
ap.add_argument("--metric", choices=sorted(METRIC_MAP.keys()), default="holdout_auc",
|
||||
help="Performance metric to analyze (default: holdout_auc)")
|
||||
ap.add_argument("--sort-by", choices=["score", "abs_rho", "rho", "r", "eta"], default="score",
|
||||
help="Sorting key for results (default: score)")
|
||||
ap.add_argument("--asc", action="store_true",
|
||||
help="Sort ascending (default: descending)")
|
||||
ap.add_argument("--desc", action="store_true",
|
||||
help="Sort descending (default: descending)")
|
||||
ap.add_argument("--top", type=int, default=30,
|
||||
help="Limit output to top N parameters (default: 30)")
|
||||
ap.add_argument("--params", default=None,
|
||||
help="Comma-separated list of parameter keys to include")
|
||||
ap.add_argument("--exclude", default=None,
|
||||
help="Comma-separated list of parameter keys to exclude")
|
||||
ap.add_argument("--min-count", type=int, default=10,
|
||||
help="Minimum runs required to analyze a parameter (default: 10)")
|
||||
ap.add_argument("--min-unique", type=int, default=2,
|
||||
help="Minimum unique values required (default: 2)")
|
||||
ap.add_argument("--match", default=None,
|
||||
help="Only include run directories whose name contains this substring")
|
||||
ap.add_argument("--shallow", action="store_true",
|
||||
help="Only scan directories directly under analysis-dir")
|
||||
ap.add_argument("--no-progress", action="store_true",
|
||||
help="Disable progress output")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.asc and args.desc:
|
||||
raise SystemExit("Choose only one of --asc or --desc.")
|
||||
|
||||
root = args.analysis_dir
|
||||
if not root.exists():
|
||||
raise SystemExit(f"Analysis directory not found: {root}")
|
||||
|
||||
exclude_keys = set(DEFAULT_EXCLUDE_KEYS)
|
||||
if args.exclude:
|
||||
for item in args.exclude.split(","):
|
||||
item = item.strip()
|
||||
if item:
|
||||
exclude_keys.add(item)
|
||||
|
||||
runs: List[Dict[str, object]] = []
|
||||
values_by_key: Dict[str, List[object]] = {}
|
||||
missing_summary = 0
|
||||
unknown_task = 0
|
||||
missing_cli = 0
|
||||
|
||||
metric_key = METRIC_MAP[args.metric]
|
||||
|
||||
for run_dir in iter_run_dirs(root, shallow=args.shallow, show_progress=not args.no_progress):
|
||||
if args.match and args.match not in run_dir.name:
|
||||
continue
|
||||
summary = read_summary(run_dir)
|
||||
if summary is None:
|
||||
missing_summary += 1
|
||||
continue
|
||||
task_label = task_from_summary(summary)
|
||||
if args.task != "all":
|
||||
if task_label is None:
|
||||
unknown_task += 1
|
||||
continue
|
||||
if task_label != args.task:
|
||||
continue
|
||||
|
||||
metric_value = mean_metric(summary, metric_key)
|
||||
if metric_value is None:
|
||||
continue
|
||||
|
||||
cli_args = read_cli_args(run_dir)
|
||||
if cli_args is None:
|
||||
missing_cli += 1
|
||||
config_source = cli_args if cli_args is not None else summary
|
||||
config = flatten_config(config_source, exclude_keys=exclude_keys)
|
||||
|
||||
run_id = read_run_id(run_dir, summary)
|
||||
runs.append({
|
||||
"run_id": run_id,
|
||||
"metric": metric_value,
|
||||
"config": config,
|
||||
})
|
||||
for key, value in config.items():
|
||||
values_by_key.setdefault(key, []).append(value)
|
||||
|
||||
if not runs:
|
||||
print("No matching runs found.")
|
||||
return
|
||||
|
||||
if args.params:
|
||||
param_keys = [p.strip() for p in args.params.split(",") if p.strip()]
|
||||
else:
|
||||
param_keys = []
|
||||
for key, values in values_by_key.items():
|
||||
unique_values = {format_value(v) for v in values}
|
||||
if len(unique_values) >= args.min_unique:
|
||||
param_keys.append(key)
|
||||
param_keys.sort()
|
||||
|
||||
rows: List[Dict[str, object]] = []
|
||||
for key in param_keys:
|
||||
values = []
|
||||
metrics = []
|
||||
for run in runs:
|
||||
config = run.get("config", {})
|
||||
if key not in config:
|
||||
continue
|
||||
values.append(config[key])
|
||||
metrics.append(run["metric"])
|
||||
|
||||
if len(values) < args.min_count:
|
||||
continue
|
||||
|
||||
unique_values = {format_value(v) for v in values}
|
||||
if len(unique_values) < args.min_unique:
|
||||
continue
|
||||
|
||||
numeric_values: List[float] = []
|
||||
numeric_ok = True
|
||||
for v in values:
|
||||
num = to_float(v)
|
||||
if num is None or isinstance(v, bool):
|
||||
numeric_ok = False
|
||||
break
|
||||
numeric_values.append(num)
|
||||
|
||||
groups: Dict[object, List[float]] = {}
|
||||
for val, metric in zip(values, metrics):
|
||||
groups.setdefault(val, []).append(metric)
|
||||
group_means = {k: mean(v) for k, v in groups.items()}
|
||||
best_group = max(group_means.items(), key=lambda item: item[1] or float("-inf"))
|
||||
worst_group = min(group_means.items(), key=lambda item: item[1] or float("inf"))
|
||||
|
||||
if numeric_ok and len(set(numeric_values)) >= 3:
|
||||
rho = spearman(numeric_values, metrics)
|
||||
r = pearson(numeric_values, metrics)
|
||||
score = abs(rho) if rho is not None else None
|
||||
row = {
|
||||
"param": key,
|
||||
"type": "numeric",
|
||||
"n": len(values),
|
||||
"distinct": len(unique_values),
|
||||
"score": format_metric(score) if score is not None else "",
|
||||
"rho": format_metric(rho),
|
||||
"r": format_metric(r),
|
||||
"best_value": format_value(best_group[0]),
|
||||
"best_mean": format_metric(best_group[1]),
|
||||
"worst_value": format_value(worst_group[0]),
|
||||
"worst_mean": format_metric(worst_group[1]),
|
||||
}
|
||||
else:
|
||||
eta = correlation_ratio(values, metrics)
|
||||
score = eta
|
||||
row = {
|
||||
"param": key,
|
||||
"type": "categorical",
|
||||
"n": len(values),
|
||||
"distinct": len(unique_values),
|
||||
"score": format_metric(score) if score is not None else "",
|
||||
"rho": "",
|
||||
"r": "",
|
||||
"best_value": format_value(best_group[0]),
|
||||
"best_mean": format_metric(best_group[1]),
|
||||
"worst_value": format_value(worst_group[0]),
|
||||
"worst_mean": format_metric(worst_group[1]),
|
||||
}
|
||||
|
||||
rows.append(row)
|
||||
|
||||
if not rows:
|
||||
print("No parameters met the minimum requirements.")
|
||||
return
|
||||
|
||||
def sort_key(row: Dict[str, object]) -> float:
|
||||
raw = row.get(args.sort_by)
|
||||
if isinstance(raw, str):
|
||||
val = to_float(raw)
|
||||
else:
|
||||
val = to_float(raw)
|
||||
if val is None:
|
||||
return float("inf") if args.asc else float("-inf")
|
||||
return float(val)
|
||||
|
||||
rows.sort(key=sort_key, reverse=not args.asc)
|
||||
if args.top is not None:
|
||||
rows = rows[:args.top]
|
||||
|
||||
header_lines = []
|
||||
header_lines.append(f"Metric: {args.metric} (mean over folds)")
|
||||
if args.task != "all":
|
||||
header_lines.append(f"Task filter: {args.task}")
|
||||
if args.match:
|
||||
header_lines.append(f"Name filter: {args.match}")
|
||||
header_lines.append(f"Runs: {len(runs)}")
|
||||
if missing_cli:
|
||||
header_lines.append(f"Missing cli_args: {missing_cli}")
|
||||
if missing_summary or unknown_task:
|
||||
header_lines.append(f"Skipped: {missing_summary} missing summary, {unknown_task} unknown task")
|
||||
header_lines.append("")
|
||||
print("\n".join(header_lines))
|
||||
|
||||
columns = [
|
||||
"param",
|
||||
"type",
|
||||
"n",
|
||||
"distinct",
|
||||
"score",
|
||||
"rho",
|
||||
"r",
|
||||
"best_value",
|
||||
"best_mean",
|
||||
"worst_value",
|
||||
"worst_mean",
|
||||
]
|
||||
print(format_table(rows, columns))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Re-run a single grid-search configuration into analysis_data/re_runs."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
|
||||
# ---------------------------
|
||||
# Config (edit in IDE)
|
||||
# ---------------------------
|
||||
RUN_ID = "20251129-0063" # fallback if --run-number is not provided
|
||||
OUTPUT_RUN_ID = RUN_ID # fallback output run id
|
||||
SHORTNAME = "re_runs" # output root under analysis_data/ and models/
|
||||
GRID_PLAN = Path("analysis_data/grid_search/grid_plan.csv")
|
||||
MANIFEST = Path("manifest.csv")
|
||||
RUN_SCRIPT = Path("scripts/run_multifold.py")
|
||||
REBUILD_SCRIPT = Path("scripts/rebuild_run_best_plots.py")
|
||||
PLOT_HEADS = ["fused", "image", "metadata"]
|
||||
USE_HOLDOUT_BEST_FOR_PLOTS = True
|
||||
OVERWRITE_HOLDOUT_PROBS = True
|
||||
ALLOW_EXISTING_RUN_DIR = False
|
||||
|
||||
|
||||
def _parse_args() -> argparse.Namespace:
|
||||
ap = argparse.ArgumentParser(description="Re-run a single grid-search item.")
|
||||
ap.add_argument(
|
||||
"--run-number",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Last 4 digits of run_id (e.g., 0063).",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--output-run-id",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Optional output run id; defaults to matched run_id.",
|
||||
)
|
||||
return ap.parse_args()
|
||||
|
||||
|
||||
def _read_plan(path: Path) -> List[Dict[str, str]]:
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Grid plan not found: {path}")
|
||||
with path.open(newline="") as fh:
|
||||
reader = csv.DictReader(fh)
|
||||
return list(reader)
|
||||
|
||||
|
||||
def _find_row(rows: List[Dict[str, str]], run_id: str) -> Dict[str, str]:
|
||||
for row in rows:
|
||||
if row.get("run_id") == run_id:
|
||||
return row
|
||||
raise ValueError(f"run_id not found in grid plan: {run_id}")
|
||||
|
||||
|
||||
def _resolve_run_id(rows: List[Dict[str, str]], run_number: str | None) -> str:
|
||||
if not run_number:
|
||||
return RUN_ID
|
||||
run_number = str(run_number).strip()
|
||||
if run_number.isdigit():
|
||||
run_number = run_number.zfill(4)
|
||||
matches = [
|
||||
r.get("run_id", "")
|
||||
for r in rows
|
||||
if str(r.get("run_id", "")).endswith(f"-{run_number}")
|
||||
]
|
||||
if len(matches) == 1:
|
||||
return matches[0]
|
||||
if len(matches) > 1:
|
||||
raise ValueError(
|
||||
f"Multiple run_ids matched run-number '{run_number}': {matches[:5]}{' ...' if len(matches) > 5 else ''}"
|
||||
)
|
||||
raise ValueError(f"No run_id found ending with '-{run_number}'")
|
||||
|
||||
|
||||
def _build_run_command(row: Dict[str, str], output_run_id: str) -> List[str]:
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(RUN_SCRIPT),
|
||||
"--backbone",
|
||||
"resnet50",
|
||||
"--fusion-mode",
|
||||
"fused",
|
||||
"--epochs",
|
||||
"40",
|
||||
"--batch-size",
|
||||
"8",
|
||||
"--img-crop-manifest",
|
||||
str(MANIFEST),
|
||||
"--img-crop-weights",
|
||||
row["crop_weights"],
|
||||
"--img-crop-normalize",
|
||||
row["crop_normalize"],
|
||||
"--eval_mode",
|
||||
row["eval_mode"],
|
||||
"--holdout-per-class",
|
||||
"12",
|
||||
"--run-id",
|
||||
output_run_id,
|
||||
"--shortname",
|
||||
SHORTNAME,
|
||||
]
|
||||
if row.get("crop_tta") == "True":
|
||||
cmd.append("--img-crop-tta")
|
||||
|
||||
loss_mode = row.get("loss_mode")
|
||||
if loss_mode == "focal":
|
||||
cmd.extend(["--focal-gamma", "2.0"])
|
||||
elif loss_mode == "balanced":
|
||||
cmd.append("--balanced-sampler")
|
||||
|
||||
thaw_mode = row.get("thaw_mode")
|
||||
if thaw_mode == "gradual":
|
||||
cmd.append("--gradual-thaw")
|
||||
cmd.extend(["--thaw-ratio", "0.33"])
|
||||
cmd.extend(["--thaw-start-epoch", "10"])
|
||||
cmd.extend(["--thaw-target", "image"])
|
||||
|
||||
se_mode = row.get("se_mode")
|
||||
if se_mode == "none":
|
||||
cmd.append("--no-se")
|
||||
else:
|
||||
cmd.extend(["--se-reduction", "16"])
|
||||
cmd.extend(["--se-reduction-tower", "16"])
|
||||
cmd.extend(["--se-where", se_mode])
|
||||
bridge_pre = row.get("se_bridge_pre_norm")
|
||||
tower_pre = row.get("se_tower_pre_norm")
|
||||
if bridge_pre == "True":
|
||||
cmd.append("--se-pre-norm")
|
||||
elif bridge_pre == "False":
|
||||
cmd.append("--no-se-pre-norm")
|
||||
if tower_pre == "True":
|
||||
cmd.append("--se-pre-norm-tower")
|
||||
elif tower_pre == "False":
|
||||
cmd.append("--no-se-pre-norm-tower")
|
||||
|
||||
return cmd
|
||||
|
||||
|
||||
def _swap_in_holdout_best(models_dir: Path) -> None:
|
||||
for fold_dir in sorted(models_dir.glob("fold*")):
|
||||
if not fold_dir.is_dir():
|
||||
continue
|
||||
holdout_best = fold_dir / "model_holdout_best.pt"
|
||||
model_best = fold_dir / "model_best.pt"
|
||||
if not holdout_best.exists():
|
||||
print(f"[warn] {holdout_best} missing; skipping.")
|
||||
continue
|
||||
if model_best.exists():
|
||||
backup = fold_dir / "model_best_from_train.pt"
|
||||
if not backup.exists():
|
||||
try:
|
||||
shutil.copy2(model_best, backup)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
shutil.copy2(holdout_best, model_best)
|
||||
except Exception as exc:
|
||||
print(f"[warn] failed to replace {model_best}: {exc}")
|
||||
|
||||
|
||||
def _run_rebuild(run_dir: Path) -> None:
|
||||
for head in PLOT_HEADS:
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(REBUILD_SCRIPT),
|
||||
"--run-dir",
|
||||
str(run_dir),
|
||||
"--head",
|
||||
head,
|
||||
"--use-holdout",
|
||||
]
|
||||
if OVERWRITE_HOLDOUT_PROBS:
|
||||
cmd.append("--overwrite")
|
||||
print("[rerun] Rebuilding holdout ROC plots:", " ".join(cmd))
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = _parse_args()
|
||||
rows = _read_plan(GRID_PLAN)
|
||||
run_id = _resolve_run_id(rows, args.run_number)
|
||||
row = _find_row(rows, run_id)
|
||||
output_run_id = args.output_run_id or run_id
|
||||
|
||||
run_dir = Path("analysis_data") / SHORTNAME / output_run_id
|
||||
if run_dir.exists() and not ALLOW_EXISTING_RUN_DIR:
|
||||
raise SystemExit(
|
||||
f"Run directory already exists: {run_dir} (set ALLOW_EXISTING_RUN_DIR=True to reuse)"
|
||||
)
|
||||
|
||||
cmd = _build_run_command(row, output_run_id=output_run_id)
|
||||
print("[rerun] Launching:", " ".join(cmd))
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
models_dir = Path("models") / SHORTNAME / output_run_id
|
||||
if USE_HOLDOUT_BEST_FOR_PLOTS:
|
||||
print("[rerun] Swapping in holdout-best checkpoints for plotting.")
|
||||
_swap_in_holdout_best(models_dir)
|
||||
|
||||
_run_rebuild(run_dir)
|
||||
print(f"[rerun] Done. Outputs in {run_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,215 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Re-run a single grid-search configuration using the V2 loader pipeline."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
|
||||
# ---------------------------
|
||||
# Config (edit in IDE)
|
||||
# ---------------------------
|
||||
RUN_ID = "20251129-0063" # fallback if --run-number is not provided
|
||||
OUTPUT_RUN_ID = RUN_ID # fallback output run id
|
||||
SHORTNAME = "re_runs_v2" # output root under analysis_data/ and models/
|
||||
GRID_PLAN = Path("analysis_data/grid_search/grid_plan.csv")
|
||||
MANIFEST = Path("manifest.csv")
|
||||
RUN_SCRIPT = Path("scripts/run_multifold_v2.py")
|
||||
REBUILD_SCRIPT = Path("scripts/rebuild_run_best_plots.py")
|
||||
PLOT_HEADS = ["fused", "image", "metadata"]
|
||||
USE_HOLDOUT_BEST_FOR_PLOTS = True
|
||||
OVERWRITE_HOLDOUT_PROBS = True
|
||||
ALLOW_EXISTING_RUN_DIR = False
|
||||
SAMPLE_MODE = "eye" # eye-level for parity with v1 grid runs
|
||||
|
||||
|
||||
def _parse_args() -> argparse.Namespace:
|
||||
ap = argparse.ArgumentParser(description="Re-run a single grid-search item with V2 loaders.")
|
||||
ap.add_argument(
|
||||
"--run-number",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Last 4 digits of run_id (e.g., 0063).",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--output-run-id",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Optional output run id; defaults to matched run_id.",
|
||||
)
|
||||
return ap.parse_args()
|
||||
|
||||
|
||||
def _read_plan(path: Path) -> List[Dict[str, str]]:
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Grid plan not found: {path}")
|
||||
with path.open(newline="") as fh:
|
||||
reader = csv.DictReader(fh)
|
||||
return list(reader)
|
||||
|
||||
|
||||
def _find_row(rows: List[Dict[str, str]], run_id: str) -> Dict[str, str]:
|
||||
for row in rows:
|
||||
if row.get("run_id") == run_id:
|
||||
return row
|
||||
raise ValueError(f"run_id not found in grid plan: {run_id}")
|
||||
|
||||
|
||||
def _resolve_run_id(rows: List[Dict[str, str]], run_number: str | None) -> str:
|
||||
if not run_number:
|
||||
return RUN_ID
|
||||
run_number = str(run_number).strip()
|
||||
if run_number.isdigit():
|
||||
run_number = run_number.zfill(4)
|
||||
matches = [
|
||||
r.get("run_id", "")
|
||||
for r in rows
|
||||
if str(r.get("run_id", "")).endswith(f"-{run_number}")
|
||||
]
|
||||
if len(matches) == 1:
|
||||
return matches[0]
|
||||
if len(matches) > 1:
|
||||
raise ValueError(
|
||||
f"Multiple run_ids matched run-number '{run_number}': {matches[:5]}{' ...' if len(matches) > 5 else ''}"
|
||||
)
|
||||
raise ValueError(f"No run_id found ending with '-{run_number}'")
|
||||
|
||||
|
||||
def _build_run_command(row: Dict[str, str], output_run_id: str) -> List[str]:
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(RUN_SCRIPT),
|
||||
"--backbone",
|
||||
"resnet50",
|
||||
"--fusion-mode",
|
||||
"fused",
|
||||
"--epochs",
|
||||
"40",
|
||||
"--batch-size",
|
||||
"8",
|
||||
"--img-crop-manifest",
|
||||
str(MANIFEST),
|
||||
"--img-crop-weights",
|
||||
row["crop_weights"],
|
||||
"--img-crop-normalize",
|
||||
row["crop_normalize"],
|
||||
"--eval_mode",
|
||||
row["eval_mode"],
|
||||
"--holdout-per-class",
|
||||
"12",
|
||||
"--run-id",
|
||||
output_run_id,
|
||||
"--shortname",
|
||||
SHORTNAME,
|
||||
"--sample-mode",
|
||||
SAMPLE_MODE,
|
||||
]
|
||||
if row.get("crop_tta") == "True":
|
||||
cmd.append("--img-crop-tta")
|
||||
|
||||
loss_mode = row.get("loss_mode")
|
||||
if loss_mode == "focal":
|
||||
cmd.extend(["--focal-gamma", "2.0"])
|
||||
elif loss_mode == "balanced":
|
||||
cmd.append("--balanced-sampler")
|
||||
|
||||
thaw_mode = row.get("thaw_mode")
|
||||
if thaw_mode == "gradual":
|
||||
cmd.append("--gradual-thaw")
|
||||
cmd.extend(["--thaw-ratio", "0.33"])
|
||||
cmd.extend(["--thaw-start-epoch", "10"])
|
||||
cmd.extend(["--thaw-target", "image"])
|
||||
|
||||
se_mode = row.get("se_mode")
|
||||
if se_mode == "none":
|
||||
cmd.append("--no-se")
|
||||
else:
|
||||
cmd.extend(["--se-reduction", "16"])
|
||||
cmd.extend(["--se-reduction-tower", "16"])
|
||||
cmd.extend(["--se-where", se_mode])
|
||||
bridge_pre = row.get("se_bridge_pre_norm")
|
||||
tower_pre = row.get("se_tower_pre_norm")
|
||||
if bridge_pre == "True":
|
||||
cmd.append("--se-pre-norm")
|
||||
elif bridge_pre == "False":
|
||||
cmd.append("--no-se-pre-norm")
|
||||
if tower_pre == "True":
|
||||
cmd.append("--se-pre-norm-tower")
|
||||
elif tower_pre == "False":
|
||||
cmd.append("--no-se-pre-norm-tower")
|
||||
|
||||
return cmd
|
||||
|
||||
|
||||
def _swap_in_holdout_best(models_dir: Path) -> None:
|
||||
for fold_dir in sorted(models_dir.glob("fold*")):
|
||||
if not fold_dir.is_dir():
|
||||
continue
|
||||
holdout_best = fold_dir / "model_holdout_best.pt"
|
||||
model_best = fold_dir / "model_best.pt"
|
||||
if not holdout_best.exists():
|
||||
print(f"[warn] {holdout_best} missing; skipping.")
|
||||
continue
|
||||
if model_best.exists():
|
||||
backup = fold_dir / "model_best_from_train.pt"
|
||||
if not backup.exists():
|
||||
try:
|
||||
shutil.copy2(model_best, backup)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
shutil.copy2(holdout_best, model_best)
|
||||
except Exception as exc:
|
||||
print(f"[warn] failed to replace {model_best}: {exc}")
|
||||
|
||||
|
||||
def _run_rebuild(run_dir: Path) -> None:
|
||||
for head in PLOT_HEADS:
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(REBUILD_SCRIPT),
|
||||
"--run-dir",
|
||||
str(run_dir),
|
||||
"--head",
|
||||
head,
|
||||
"--use-holdout",
|
||||
]
|
||||
if OVERWRITE_HOLDOUT_PROBS:
|
||||
cmd.append("--overwrite")
|
||||
print("[rerun] Rebuilding holdout ROC plots:", " ".join(cmd))
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = _parse_args()
|
||||
rows = _read_plan(GRID_PLAN)
|
||||
run_id = _resolve_run_id(rows, args.run_number)
|
||||
row = _find_row(rows, run_id)
|
||||
output_run_id = args.output_run_id or run_id
|
||||
|
||||
run_dir = Path("analysis_data") / SHORTNAME / output_run_id
|
||||
if run_dir.exists() and not ALLOW_EXISTING_RUN_DIR:
|
||||
raise SystemExit(
|
||||
f"Run directory already exists: {run_dir} (set ALLOW_EXISTING_RUN_DIR=True to reuse)"
|
||||
)
|
||||
|
||||
cmd = _build_run_command(row, output_run_id=output_run_id)
|
||||
print("[rerun] Launching:", " ".join(cmd))
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
models_dir = Path("models") / SHORTNAME / output_run_id
|
||||
if USE_HOLDOUT_BEST_FOR_PLOTS:
|
||||
print("[rerun] Swapping in holdout-best checkpoints for plotting.")
|
||||
_swap_in_holdout_best(models_dir)
|
||||
|
||||
_run_rebuild(run_dir)
|
||||
print(f"[rerun] Done. Outputs in {run_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.grid_search_analytics.derived_analysis import derived_analysis
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
ap = argparse.ArgumentParser(
|
||||
description="Generate derived grid-search analytics artifacts (fusion/error + param-performance)."
|
||||
)
|
||||
ap.add_argument("--analysis-dir", default="analysis_data/grid_search")
|
||||
ap.add_argument("--mode", choices=["binary", "multiclass"], default="multiclass")
|
||||
ap.add_argument("--method", choices=["pearson", "spearman"], default="spearman")
|
||||
ap.add_argument(
|
||||
"--x-metric",
|
||||
choices=["fusion_corrections", "fusion_corrections_per_opportunity"],
|
||||
default="fusion_corrections_per_opportunity",
|
||||
help="Fusion-correlation x-axis metric for summary bar plot.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--cat-method",
|
||||
choices=["eta", "anova", "kruskal"],
|
||||
default="kruskal",
|
||||
help="Categorical-test method for param-performance correlations.",
|
||||
)
|
||||
ap.add_argument("--top-n", type=int, default=None, help="Optional cap for per-run plots.")
|
||||
ap.add_argument(
|
||||
"--recompute",
|
||||
action="store_true",
|
||||
help="Recompute from run artifacts instead of preferring cached CSVs.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--deep-scan",
|
||||
action="store_true",
|
||||
help="Scan nested directories instead of direct children only.",
|
||||
)
|
||||
return ap.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
analysis = derived_analysis(
|
||||
Path(args.analysis_dir),
|
||||
classification_mode=args.mode,
|
||||
)
|
||||
|
||||
shallow = not args.deep_scan
|
||||
existing = not args.recompute
|
||||
|
||||
analysis.identify_fusion_corrections(shallow=shallow, existing=existing)
|
||||
analysis.populate_primary_metrics(shallow=shallow, existing=existing)
|
||||
|
||||
analysis.write_fusion_corrections()
|
||||
analysis.write_fusion_errors()
|
||||
analysis.write_primary_metrics()
|
||||
|
||||
analysis.plot_fusion_corrections_errors(
|
||||
shallow=shallow, existing=existing, top_n=args.top_n
|
||||
)
|
||||
analysis.plot_conf_delta_boxplot(
|
||||
shallow=shallow, existing=existing, top_n=args.top_n
|
||||
)
|
||||
|
||||
corr_df = analysis.param_performance_correlations(
|
||||
shallow=shallow,
|
||||
existing=existing,
|
||||
method=args.method,
|
||||
cat_method=args.cat_method,
|
||||
)
|
||||
analysis.plot_param_perf_corr_panels(corr_df)
|
||||
|
||||
corr_acc = analysis.fusion_corrections_correlation(
|
||||
method=args.method, metric_type="acc"
|
||||
)
|
||||
corr_auc = analysis.fusion_corrections_correlation(
|
||||
method=args.method, metric_type="auc"
|
||||
)
|
||||
try:
|
||||
analysis.plot_fusion_perf_summary(
|
||||
corr_acc,
|
||||
corr_auc,
|
||||
method=args.method,
|
||||
x_metric=args.x_metric,
|
||||
)
|
||||
except RuntimeError:
|
||||
analysis.plot_fusion_perf_summary(
|
||||
corr_acc,
|
||||
corr_auc,
|
||||
method=args.method,
|
||||
x_metric="fusion_corrections",
|
||||
)
|
||||
|
||||
print(f"Done. Outputs written under: {Path(args.analysis_dir) / 'plots'}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user