hypertower/classes/tower_watcher.py

# tower_watcher.py
import matplotlib.pyplot as plt

# tower_watcher.py
import matplotlib.pyplot as plt


class TowerWatcher:
    """
    Live monitor:
    - Cumulative batch-level: loss & accuracy per batch across all epochs.
    - Epoch batch-level: loss & accuracy per batch within the current epoch (resets each epoch).
    - TP/FP/TN/FN bar charts per tower, one chart each, new group each epoch.
    """

    def __init__(self):
        plt.ion()
        # 2 line plots (cum loss, cum acc), 2 line plots (epoch loss, epoch acc), 3 bar plots
        self.fig, self.axs = plt.subplots(7, 1, figsize=(10, 28))
        self.reset()

    def reset(self):
        # Cumulative batch-level
        self.global_batches = []
        self.loss_cum = {"fusion": [], "image": [], "meta": []}
        self.acc_cum = {"fusion": [], "image": [], "meta": []}
        # Epoch batch-level
        self.epoch_batches = []
        self.loss_epoch_batch = {"fusion": [], "image": [], "meta": []}
        self.acc_epoch_batch = {"fusion": [], "image": [], "meta": []}
        # Epoch markers for cum plots
        self.epoch_markers = []
        # Stats per epoch for bars
        self.epoch_stats = {"fusion": [], "image": [], "meta": []}
        # Track current epoch
        self.current_epoch = -1

    def on_epoch_start(self, epoch):
        # mark epoch boundary in cumulative
        x = self.global_batches[-1] + 1 if self.global_batches else 0
        self.epoch_markers.append(x)
        # reset epoch batch-level data
        self.epoch_batches = []
        for d in [self.loss_epoch_batch, self.acc_epoch_batch]:
            for k in d:
                d[k].clear()
        self.current_epoch = epoch

    def on_batch_end(self, idx, stats: dict):
        # Cumulative
        self.global_batches.append(len(self.global_batches) + 1)
        for key, lk, ak in [
            ("fusion", "loss_f", "acc_f"),
            ("image", "loss_i", "acc_i"),
            ("meta", "loss_m", "acc_m"),
        ]:
            self.loss_cum[key].append(stats.get(lk, 0))
            self.acc_cum[key].append(stats.get(ak, 0))
        # Epoch-level
        self.epoch_batches.append(len(self.epoch_batches) + 1)
        for key, lk, ak in [
            ("fusion", "loss_f", "acc_f"),
            ("image", "loss_i", "acc_i"),
            ("meta", "loss_m", "acc_m"),
        ]:
            self.loss_epoch_batch[key].append(stats.get(lk, 0))
            self.acc_epoch_batch[key].append(stats.get(ak, 0))
        # redraw
        self._draw_batch_plots()

    def on_epoch_end(self, epoch, stats: dict):
        # record per-epoch TP/FP/TN/FN
        for key in ["fusion", "image", "meta"]:
            self.epoch_stats[key].append(
                {
                    "tp": stats.get("tp", 0),
                    "fp": stats.get("fp", 0),
                    "tn": stats.get("tn", 0),
                    "fn": stats.get("fn", 0),
                }
            )
        self._draw_epoch_bars()

    def _draw_batch_plots(self):
        # Cumulative Loss
        ax = self.axs[0]
        ax.clear()
        ax.plot(self.global_batches, self.loss_cum["fusion"], label="Fusion")
        ax.plot(self.global_batches, self.loss_cum["image"], label="Image Tower")
        ax.plot(self.global_batches, self.loss_cum["meta"], label="MD Tower")
        for x in self.epoch_markers:
            ax.axvline(x=x, color="gray", linestyle="--")
        ax.set_ylabel("Cumulative Loss")
        ax.legend()

        # Epoch Loss
        ax = self.axs[1]
        ax.clear()
        ax.plot(self.epoch_batches, self.loss_epoch_batch["fusion"], label="Fusion")
        ax.plot(self.epoch_batches, self.loss_epoch_batch["image"], label="Image Tower")
        ax.plot(self.epoch_batches, self.loss_epoch_batch["meta"], label="MD Tower")
        ax.set_ylabel(f"Epoch {self.current_epoch+1} Loss")
        ax.set_xlabel("Batch (Epoch)")
        ax.legend()

        # Cumulative Accuracy
        ax = self.axs[2]
        ax.clear()
        ax.plot(self.global_batches, self.acc_cum["fusion"], label="Fusion")
        ax.plot(self.global_batches, self.acc_cum["image"], label="Image Tower")
        ax.plot(self.global_batches, self.acc_cum["meta"], label="MD Tower")
        for x in self.epoch_markers:
            ax.axvline(x=x, color="gray", linestyle="--")
        ax.set_ylabel("Cumulative Accuracy")
        ax.legend()

        # Epoch Accuracy
        ax = self.axs[3]
        ax.clear()
        ax.plot(self.epoch_batches, self.acc_epoch_batch["fusion"], label="Fusion")
        ax.plot(self.epoch_batches, self.acc_epoch_batch["image"], label="Image Tower")
        ax.plot(self.epoch_batches, self.acc_epoch_batch["meta"], label="MD Tower")
        ax.set_ylabel(f"Epoch {self.current_epoch+1} Accuracy")
        ax.set_xlabel("Batch (Epoch)")
        ax.legend()

        plt.pause(0.01)

    def _draw_epoch_bars(self):
        # Bar charts per tower
        for i, key in enumerate(["fusion", "image", "meta"]):
            ax = self.axs[4 + i]
            ax.clear()
            data = self.epoch_stats[key]
            epochs = list(range(1, len(data) + 1))
            tp = [d["tp"] for d in data]
            fp = [d["fp"] for d in data]
            tn = [d["tn"] for d in data]
            fn = [d["fn"] for d in data]
            width = 0.2
            ax.bar([e - width for e in epochs], tp, width, label="TP")
            ax.bar(epochs, fp, width, label="FP")
            ax.bar([e + width for e in epochs], tn, width, label="TN")
            ax.bar([e + 2 * width for e in epochs], fn, width, label="FN")
            ax.set_title(f"{key.title()} Tower Stats")
            ax.set_xlabel("Epoch")
            ax.set_ylabel("Count")
            ax.legend()
        plt.pause(0.01)