moved_repo_first_update

2026-02-24 10:39:48 +01:00
commit 9894a23f09
98 changed files with 35387 additions and 0 deletions
@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+import csv
+import json
+import logging
+from pathlib import Path
+from typing import Optional
+
+
+DEFAULT_OPTIONAL_EPOCH_COLS = [
+    "pct_fused",
+    "pct_img",
+    "pct_md",
+    "phase",
+    "se_mean",
+    "se_std",
+    "se_pct_lt_0.2",
+    "se_pct_gt_0.8",
+    "holdout_loss",
+    "holdout_acc_fused",
+    "holdout_acc_img",
+    "holdout_acc_md",
+    "holdout_auc_fused",
+    "holdout_auc_img",
+    "holdout_auc_md",
+    "best_monitor",
+    "best_so_far",
+    "best_epoch",
+    "early_best_so_far",
+    "early_bad_epochs",
+    "early_improved",
+    "early_monitor",
+    "holdout_best_monitor",
+    "holdout_best_so_far",
+    "holdout_best_epoch",
+]
+
+
+class HypertowerLogger:
+    """
+    Shared logging utility for V2 tower workflows.
+    - train.log line logging
+    - epoch_log.csv row logging with stable header
+    - lightweight JSON/array artifact helpers
+    """
+
+    def __init__(
+        self,
+        *,
+        run_dir: Path,
+        train_log_path: Optional[Path] = None,
+        epoch_log_path: Optional[Path] = None,
+        logger_name: Optional[str] = None,
+    ) -> None:
+        self.run_dir = Path(run_dir).resolve()
+        self.run_dir.mkdir(parents=True, exist_ok=True)
+        self.train_log_path = Path(train_log_path) if train_log_path else (self.run_dir / "train.log")
+        self.epoch_log_path = Path(epoch_log_path) if epoch_log_path else (self.run_dir / "epoch_log.csv")
+
+        self._logger_name = logger_name or f"hypertower.{id(self)}"
+        self.logger = logging.getLogger(self._logger_name)
+        self.logger.setLevel(logging.INFO)
+        self.logger.handlers = []
+        fh = logging.FileHandler(str(self.train_log_path))
+        fh.setFormatter(logging.Formatter("%(asctime)s - %(message)s"))
+        self.logger.addHandler(fh)
+        self.logger.propagate = False
+
+        self._epoch_log_fp = None
+        self._epoch_log_writer = None
+        self._epoch_log_fields: list[str] | None = None
+
+    def info(self, msg: str) -> None:
+        self.logger.info(msg)
+
+    def warning(self, msg: str) -> None:
+        self.logger.warning(msg)
+
+    def error(self, msg: str) -> None:
+        self.logger.error(msg)
+
+    def write_epoch_row(
+        self,
+        row: dict,
+        *,
+        path: str | Path | None = None,
+        optional_cols: Optional[list[str]] = None,
+    ) -> None:
+        optional = optional_cols if optional_cols is not None else DEFAULT_OPTIONAL_EPOCH_COLS
+        if self._epoch_log_writer is None:
+            fieldnames = list(dict.fromkeys([*row.keys(), *optional]))
+            target_path = Path(path) if path is not None else self.epoch_log_path
+            target_path.parent.mkdir(parents=True, exist_ok=True)
+            self._epoch_log_fp = open(target_path, "w", newline="", encoding="utf-8")
+            self._epoch_log_writer = csv.DictWriter(self._epoch_log_fp, fieldnames=fieldnames)
+            self._epoch_log_writer.writeheader()
+            self._epoch_log_fields = fieldnames
+
+        assert self._epoch_log_fields is not None
+        assert self._epoch_log_writer is not None
+        assert self._epoch_log_fp is not None
+        for key in self._epoch_log_fields:
+            row.setdefault(key, None)
+        self._epoch_log_writer.writerow({k: row.get(k) for k in self._epoch_log_fields})
+        self._epoch_log_fp.flush()
+
+    def write_json(self, path: str | Path, payload: dict) -> None:
+        target = Path(path)
+        if not target.is_absolute():
+            target = self.run_dir / target
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+
+    def close(self) -> None:
+        if self._epoch_log_fp is not None:
+            try:
+                self._epoch_log_fp.close()
+            except Exception:
+                pass
+            self._epoch_log_fp = None
+            self._epoch_log_writer = None
+            self._epoch_log_fields = None
+        for handler in list(self.logger.handlers):
+            try:
+                handler.close()
+            except Exception:
+                pass
+            self.logger.removeHandler(handler)