rpotter6298
ccd4e7a48c
The forecasting runtime gibil pilots: weather ingest (Open-Meteo forecasts and the Ecowitt station), the oracle and weather stores, the snapshot daemon, the forecast evaluator, and the TCN training scheduler. classes/datasets.py holds the shapes the stores return for display. They sit beside the stores deliberately: a store's return type belongs to the store, and keeping them with the renderers would mean this repo importing the dashboard, which is backwards. Extracted from a single repository where the forecasting engine, the database plumbing, the hardware client, the ingest daemons and the web UI all imported each other freely. Verified that every module imports with gibil resolved from its own installed checkout. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
103 lines
3.1 KiB
Python
103 lines
3.1 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
from datetime import timedelta
|
|
|
|
from astrape_ems.classes.env import EnvLoader
|
|
from astrape_oracle.classes.oracle_store import OracleStore
|
|
|
|
|
|
def main() -> None:
|
|
EnvLoader().load()
|
|
args = parse_args()
|
|
store = OracleStore.from_env()
|
|
|
|
if args.evaluate:
|
|
evaluated_count = store.evaluate_due_forecasts(
|
|
actual_window=timedelta(minutes=args.actual_window_minutes),
|
|
lookback=timedelta(hours=args.lookback_hours),
|
|
limit=args.limit,
|
|
)
|
|
print(f"evaluated_oracle_forecasts={evaluated_count}")
|
|
|
|
if args.summary:
|
|
rows = store.load_evaluation_summary(
|
|
lookback=timedelta(hours=args.lookback_hours)
|
|
)
|
|
print(f"oracle_evaluation_summary_rows={len(rows)}")
|
|
for row in rows:
|
|
print(
|
|
" ".join(
|
|
[
|
|
f"kind={row['kind']}",
|
|
f"model={row['model_version']}",
|
|
f"horizon={row.get('horizon_label') or _format_horizon(row)}",
|
|
f"n={row['evaluated_count']}",
|
|
f"bias={_format_w(row['mean_error_w'])}",
|
|
f"mae={_format_w(row['mean_absolute_error_w'])}",
|
|
f"median_ae={_format_w(row['median_absolute_error_w'])}",
|
|
f"mape={_format_pct(row['mean_absolute_pct_error'])}",
|
|
f"coverage={_format_pct(row['interval_coverage'])}",
|
|
]
|
|
)
|
|
)
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Evaluate stored Astrape oracle predictions against Sigen actuals."
|
|
)
|
|
parser.add_argument(
|
|
"--evaluate",
|
|
action="store_true",
|
|
help="Evaluate stored predictions whose target time has passed.",
|
|
)
|
|
parser.add_argument(
|
|
"--summary",
|
|
action="store_true",
|
|
help="Print evaluation quality by kind/model/horizon bucket.",
|
|
)
|
|
parser.add_argument(
|
|
"--actual-window-minutes",
|
|
type=float,
|
|
default=5,
|
|
help="Minutes after each target timestamp to average as realized actuals.",
|
|
)
|
|
parser.add_argument(
|
|
"--lookback-hours",
|
|
type=float,
|
|
default=168,
|
|
help="Only evaluate/summarize predictions with target times this recent.",
|
|
)
|
|
parser.add_argument(
|
|
"--limit",
|
|
type=int,
|
|
default=1000,
|
|
help="Maximum unevaluated predictions to process.",
|
|
)
|
|
args = parser.parse_args()
|
|
if not args.evaluate and not args.summary:
|
|
args.evaluate = True
|
|
args.summary = True
|
|
return args
|
|
|
|
|
|
def _format_w(value: object) -> str:
|
|
if value is None:
|
|
return "n/a"
|
|
return f"{float(value):.0f}W"
|
|
|
|
|
|
def _format_horizon(row: dict[str, object]) -> str:
|
|
return f"{row['min_horizon_minutes']}-{row['max_horizon_minutes']}m"
|
|
|
|
|
|
def _format_pct(value: object) -> str:
|
|
if value is None:
|
|
return "n/a"
|
|
return f"{float(value) * 100:.1f}%"
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|