Initial commit: astrape-oracle split out of the combined Astrape tree
The forecasting runtime gibil pilots: weather ingest (Open-Meteo forecasts and the Ecowitt station), the oracle and weather stores, the snapshot daemon, the forecast evaluator, and the TCN training scheduler. classes/datasets.py holds the shapes the stores return for display. They sit beside the stores deliberately: a store's return type belongs to the store, and keeping them with the renderers would mean this repo importing the dashboard, which is backwards. Extracted from a single repository where the forecasting engine, the database plumbing, the hardware client, the ingest daemons and the web UI all imported each other freely. Verified that every module imports with gibil resolved from its own installed checkout. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
rpotter6298
@@ -0,0 +1,122 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
from os import environ
|
||||
from sys import stderr
|
||||
from time import sleep
|
||||
|
||||
from gibil.classes.oracle_builder import EnergyOracleBuilder
|
||||
from gibil.classes.forecast_config import EnergyForecastConfig
|
||||
from astrape_ems.classes.env import EnvLoader
|
||||
from astrape_oracle.classes.oracle_store import OracleStore
|
||||
from astrape_ems.classes.store import SigenStore
|
||||
from astrape_oracle.classes.weather.store import WeatherStore
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OracleDaemonConfig:
|
||||
poll_seconds: float
|
||||
evaluate_forecasts: bool
|
||||
evaluation_actual_window_minutes: float
|
||||
evaluation_lookback_hours: float
|
||||
evaluation_limit: int
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "OracleDaemonConfig":
|
||||
return cls(
|
||||
poll_seconds=float(environ.get("ASTRAPE_ORACLE_POLL_SECONDS", "300")),
|
||||
evaluate_forecasts=environ.get(
|
||||
"ASTRAPE_ORACLE_EVALUATE_FORECASTS", "1"
|
||||
).lower()
|
||||
not in {"0", "false", "no"},
|
||||
evaluation_actual_window_minutes=float(
|
||||
environ.get("ASTRAPE_ORACLE_EVALUATION_WINDOW_MINUTES", "5")
|
||||
),
|
||||
evaluation_lookback_hours=float(
|
||||
environ.get("ASTRAPE_ORACLE_EVALUATION_LOOKBACK_HOURS", "168")
|
||||
),
|
||||
evaluation_limit=int(environ.get("ASTRAPE_ORACLE_EVALUATION_LIMIT", "1000")),
|
||||
)
|
||||
|
||||
|
||||
class OracleDaemon:
|
||||
"""Periodically stores oracle projection curves for evaluation."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: OracleDaemonConfig,
|
||||
builder: EnergyOracleBuilder,
|
||||
store: OracleStore,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.builder = builder
|
||||
self.store = store
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "OracleDaemon":
|
||||
return cls(
|
||||
config=OracleDaemonConfig.from_env(),
|
||||
builder=EnergyOracleBuilder(
|
||||
weather_history=WeatherStore.from_env(),
|
||||
plant_history=SigenStore.from_env(),
|
||||
config=EnergyForecastConfig.from_env(),
|
||||
),
|
||||
store=OracleStore.from_env(),
|
||||
)
|
||||
|
||||
def run_once(self) -> int:
|
||||
solar_run, load_run, net_run = self.builder.build()
|
||||
saved_count = self.store.save_runs(solar_run, load_run, net_run)
|
||||
if self.config.evaluate_forecasts:
|
||||
from datetime import timedelta
|
||||
|
||||
evaluated_count = self.store.evaluate_due_forecasts(
|
||||
actual_window=timedelta(
|
||||
minutes=self.config.evaluation_actual_window_minutes
|
||||
),
|
||||
lookback=timedelta(hours=self.config.evaluation_lookback_hours),
|
||||
limit=self.config.evaluation_limit,
|
||||
)
|
||||
return saved_count + evaluated_count
|
||||
return saved_count
|
||||
|
||||
def run_forever(self) -> None:
|
||||
self.store.initialize()
|
||||
while True:
|
||||
try:
|
||||
saved_count = self.run_once()
|
||||
print(f"stored_oracle_records={saved_count}", flush=True)
|
||||
except Exception as error:
|
||||
print(f"oracle_poll_error={error}", file=stderr, flush=True)
|
||||
sleep(self.config.poll_seconds)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
EnvLoader().load()
|
||||
args = parse_args()
|
||||
daemon = OracleDaemon.from_env()
|
||||
if args.once:
|
||||
print(f"stored_oracle_records={daemon.run_once()}", flush=True)
|
||||
return
|
||||
daemon.run_forever()
|
||||
except Exception as error:
|
||||
print(f"oracle_daemon_startup_error={error}", file=stderr)
|
||||
raise SystemExit(1) from error
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Store Astrape oracle projection curves."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--once",
|
||||
action="store_true",
|
||||
help="Save one set of oracle curves and exit.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,137 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from os import environ
|
||||
from sys import stderr
|
||||
from time import sleep
|
||||
|
||||
from astrape_ems.classes.env import EnvLoader
|
||||
from astrape_oracle.classes.weather.openmeteo import (
|
||||
OpenMeteoArchiveClient,
|
||||
OpenMeteoClient,
|
||||
WeatherBuilder,
|
||||
)
|
||||
from astrape_oracle.classes.weather.store import WeatherStore
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DbDaemonConfig:
|
||||
latitude: float
|
||||
longitude: float
|
||||
forecast_hours: int
|
||||
truth_lookback_days: int
|
||||
truth_end_delay_days: int
|
||||
poll_seconds: int
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "DbDaemonConfig":
|
||||
return cls(
|
||||
latitude=float(_required_env("ASTRAPE_LATITUDE")),
|
||||
longitude=float(_required_env("ASTRAPE_LONGITUDE")),
|
||||
forecast_hours=int(environ.get("ASTRAPE_WEATHER_FORECAST_HOURS", "48")),
|
||||
truth_lookback_days=int(
|
||||
environ.get("ASTRAPE_WEATHER_TRUTH_LOOKBACK_DAYS", "14")
|
||||
),
|
||||
truth_end_delay_days=int(
|
||||
environ.get("ASTRAPE_WEATHER_TRUTH_END_DELAY_DAYS", "5")
|
||||
),
|
||||
poll_seconds=int(environ.get("ASTRAPE_WEATHER_POLL_SECONDS", "3600")),
|
||||
)
|
||||
|
||||
|
||||
class DbDaemon:
|
||||
"""Runs builder components that populate Astrape's database."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: DbDaemonConfig,
|
||||
weather_client: OpenMeteoClient,
|
||||
archive_client: OpenMeteoArchiveClient,
|
||||
weather_builder: WeatherBuilder,
|
||||
weather_store: WeatherStore,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.weather_client = weather_client
|
||||
self.archive_client = archive_client
|
||||
self.weather_builder = weather_builder
|
||||
self.weather_store = weather_store
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "DbDaemon":
|
||||
return cls(
|
||||
config=DbDaemonConfig.from_env(),
|
||||
weather_client=OpenMeteoClient(),
|
||||
archive_client=OpenMeteoArchiveClient(),
|
||||
weather_builder=WeatherBuilder(),
|
||||
weather_store=WeatherStore.from_env(),
|
||||
)
|
||||
|
||||
def initialize(self) -> None:
|
||||
self.weather_store.initialize()
|
||||
|
||||
def run_once(self) -> tuple[int, int]:
|
||||
raw_run = self.weather_client.fetch_forecast(
|
||||
latitude=self.config.latitude,
|
||||
longitude=self.config.longitude,
|
||||
forecast_hours=self.config.forecast_hours,
|
||||
)
|
||||
forecast_run = self.weather_builder.build_forecast_run(
|
||||
source=raw_run.source,
|
||||
latitude=raw_run.latitude,
|
||||
longitude=raw_run.longitude,
|
||||
points=raw_run.points,
|
||||
issued_at=raw_run.issued_at,
|
||||
)
|
||||
forecast_count = self.weather_store.save_forecast_run(forecast_run)
|
||||
zero_hour_truth_count = self.weather_store.save_zero_hour_forecast_as_truth(
|
||||
forecast_run
|
||||
)
|
||||
|
||||
today = datetime.now(timezone.utc).date()
|
||||
truth_end = today - timedelta(days=self.config.truth_end_delay_days)
|
||||
truth_start = truth_end - timedelta(days=self.config.truth_lookback_days)
|
||||
truth_points = self.archive_client.fetch_resolved_truth(
|
||||
latitude=self.config.latitude,
|
||||
longitude=self.config.longitude,
|
||||
start_date=truth_start,
|
||||
end_date=truth_end,
|
||||
)
|
||||
archive_truth_count = self.weather_store.save_resolved_truth(truth_points)
|
||||
|
||||
return forecast_count, zero_hour_truth_count + archive_truth_count
|
||||
|
||||
def run_forever(self) -> None:
|
||||
self.initialize()
|
||||
while True:
|
||||
forecast_count, truth_count = self.run_once()
|
||||
print(
|
||||
f"stored_weather_forecast_points={forecast_count} "
|
||||
f"stored_weather_resolved_truth={truth_count}",
|
||||
flush=True,
|
||||
)
|
||||
sleep(self.config.poll_seconds)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
EnvLoader().load()
|
||||
daemon = DbDaemon.from_env()
|
||||
daemon.run_forever()
|
||||
except Exception as error:
|
||||
print(f"db_daemon_startup_error={error}", file=stderr)
|
||||
raise SystemExit(1) from error
|
||||
|
||||
|
||||
def _required_env(name: str) -> str:
|
||||
value = environ.get(name)
|
||||
if not value:
|
||||
raise RuntimeError(
|
||||
f"{name} is required. Set ASTRAPE_DATABASE_URL, "
|
||||
"ASTRAPE_LATITUDE, and ASTRAPE_LONGITUDE before starting db_daemon."
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,113 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
from os import environ
|
||||
from sys import stderr
|
||||
from time import sleep
|
||||
|
||||
from astrape_ems.classes.env import EnvLoader
|
||||
from astrape_oracle.classes.weather.ecowitt import EcowittGw3000Client
|
||||
from astrape_oracle.classes.weather.store import WeatherStore
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WeatherStationDaemonConfig:
|
||||
host: str
|
||||
poll_seconds: float
|
||||
timeout_seconds: float
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "WeatherStationDaemonConfig":
|
||||
host = environ.get("ECOWITT_GW3000_HOST")
|
||||
if not host:
|
||||
raise RuntimeError("ECOWITT_GW3000_HOST is required")
|
||||
|
||||
return cls(
|
||||
host=host,
|
||||
poll_seconds=float(environ.get("ECOWITT_GW3000_POLL_SECONDS", "60")),
|
||||
timeout_seconds=float(environ.get("ECOWITT_GW3000_TIMEOUT_SECONDS", "5")),
|
||||
)
|
||||
|
||||
|
||||
class WeatherStationDaemon:
|
||||
"""Polls local weather station observations into TimescaleDB."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WeatherStationDaemonConfig,
|
||||
client: EcowittGw3000Client,
|
||||
weather_store: WeatherStore,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.client = client
|
||||
self.weather_store = weather_store
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "WeatherStationDaemon":
|
||||
config = WeatherStationDaemonConfig.from_env()
|
||||
return cls(
|
||||
config=config,
|
||||
client=EcowittGw3000Client(
|
||||
host=config.host,
|
||||
timeout_seconds=config.timeout_seconds,
|
||||
),
|
||||
weather_store=WeatherStore.from_env(),
|
||||
)
|
||||
|
||||
def initialize(self) -> None:
|
||||
self.weather_store.initialize()
|
||||
|
||||
def run_once(self) -> int:
|
||||
observation = self.client.fetch_observation()
|
||||
return self.weather_store.save_station_observation(observation)
|
||||
|
||||
def run_forever(self) -> None:
|
||||
self.initialize()
|
||||
while True:
|
||||
try:
|
||||
saved_count = self.run_once()
|
||||
print(
|
||||
f"stored_weather_station_observations={saved_count}",
|
||||
flush=True,
|
||||
)
|
||||
except Exception as error:
|
||||
print(f"weather_station_poll_error={error}", file=stderr, flush=True)
|
||||
|
||||
sleep(self.config.poll_seconds)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
EnvLoader().load()
|
||||
daemon = WeatherStationDaemon.from_env()
|
||||
args = parse_args()
|
||||
if args.once:
|
||||
daemon.initialize()
|
||||
saved_count = daemon.run_once()
|
||||
print(
|
||||
f"stored_weather_station_observations={saved_count}",
|
||||
flush=True,
|
||||
)
|
||||
return
|
||||
|
||||
daemon.run_forever()
|
||||
except Exception as error:
|
||||
print(f"weather_station_daemon_startup_error={error}", file=stderr)
|
||||
raise SystemExit(1) from error
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Poll local Ecowitt weather station observations into Astrape."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--once",
|
||||
action="store_true",
|
||||
help="Initialize storage, save one observation, and exit.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Debug script to find duplicate target times in forecast data."""
|
||||
|
||||
from astrape_ems.classes.env import EnvLoader
|
||||
from astrape_oracle.classes.weather.store import WeatherStore
|
||||
from collections import defaultdict
|
||||
|
||||
EnvLoader().load()
|
||||
|
||||
store = WeatherStore.from_env()
|
||||
dataset = store.load_display_dataset()
|
||||
|
||||
# Group by (target_at, horizon_hours) to find duplicates
|
||||
by_key = defaultdict(list)
|
||||
for point in dataset.forecast_points:
|
||||
key = (point.target_at, point.horizon_hours)
|
||||
by_key[key].append(point)
|
||||
|
||||
# Find duplicates
|
||||
duplicates = {k: v for k, v in by_key.items() if len(v) > 1}
|
||||
|
||||
print(f"\nTotal forecast points: {len(dataset.forecast_points)}")
|
||||
print(f"Unique (target_at, horizon) pairs: {len(by_key)}")
|
||||
print(f"Duplicate (target_at, horizon) pairs: {len(duplicates)}")
|
||||
|
||||
if duplicates:
|
||||
print("\nFirst 3 duplicates:")
|
||||
for (target_at, horizon), points in list(duplicates.items())[:3]:
|
||||
print(f"\n target_at={target_at}, horizon={horizon}h ({len(points)} points):")
|
||||
for i, p in enumerate(points):
|
||||
print(f" [{i}] issued_at={p.issued_at}, temp={p.temperature_c}, source={p.source}")
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Debug baseline energy forecast curves."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import timezone
|
||||
|
||||
from gibil.classes.oracle_builder import EnergyForecastBuilder
|
||||
from gibil.classes.forecast_config import EnergyForecastConfig
|
||||
from astrape_ems.classes.env import EnvLoader
|
||||
from astrape_ems.classes.store import SigenStore
|
||||
from astrape_oracle.classes.weather.store import WeatherStore
|
||||
from gibil.classes.models import PowerForecastPoint
|
||||
|
||||
|
||||
def main() -> None:
|
||||
EnvLoader().load()
|
||||
args = parse_args()
|
||||
solar_run, load_run, net_run = EnergyForecastBuilder(
|
||||
weather_history=WeatherStore.from_env(),
|
||||
plant_history=SigenStore.from_env(),
|
||||
config=EnergyForecastConfig.from_env(),
|
||||
).build()
|
||||
|
||||
print(
|
||||
f"issued_at={net_run.issued_at.astimezone(timezone.utc).isoformat(timespec='seconds')}"
|
||||
)
|
||||
print(
|
||||
f"solar_model={solar_run.model_version} "
|
||||
f"load_model={load_run.model_version} points={len(net_run.points)}"
|
||||
)
|
||||
print(
|
||||
"target_at solar_p10 solar_p50 solar_p90 "
|
||||
"load_p10 load_p50 load_p90 net_p10 net_p50 net_p90"
|
||||
)
|
||||
solar_by_target = _by_target(solar_run.points)
|
||||
load_by_target = _by_target(load_run.points)
|
||||
for point in net_run.points[: args.limit]:
|
||||
solar_point = solar_by_target[point.target_at]
|
||||
load_point = load_by_target[point.target_at]
|
||||
print(
|
||||
f"{point.target_at.astimezone(timezone.utc).isoformat(timespec='minutes'):25} "
|
||||
f"{solar_point.p10_power_w:9.0f} "
|
||||
f"{solar_point.p50_power_w:9.0f} "
|
||||
f"{solar_point.p90_power_w:9.0f} "
|
||||
f"{load_point.p10_power_w:8.0f} "
|
||||
f"{load_point.p50_power_w:8.0f} "
|
||||
f"{load_point.p90_power_w:8.0f} "
|
||||
f"{point.p10_net_power_w:7.0f} "
|
||||
f"{point.p50_net_power_w:7.0f} "
|
||||
f"{point.p90_net_power_w:7.0f}"
|
||||
)
|
||||
|
||||
|
||||
def _by_target(points: list[PowerForecastPoint]) -> dict[object, PowerForecastPoint]:
|
||||
return {point.target_at: point for point in points}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Print baseline solar/load/net forecast curves."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=24,
|
||||
help="Number of forecast points to show. Defaults to 24.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,5 @@
|
||||
from astrape_oracle.scripts.evaluator import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,102 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import timedelta
|
||||
|
||||
from astrape_ems.classes.env import EnvLoader
|
||||
from astrape_oracle.classes.oracle_store import OracleStore
|
||||
|
||||
|
||||
def main() -> None:
|
||||
EnvLoader().load()
|
||||
args = parse_args()
|
||||
store = OracleStore.from_env()
|
||||
|
||||
if args.evaluate:
|
||||
evaluated_count = store.evaluate_due_forecasts(
|
||||
actual_window=timedelta(minutes=args.actual_window_minutes),
|
||||
lookback=timedelta(hours=args.lookback_hours),
|
||||
limit=args.limit,
|
||||
)
|
||||
print(f"evaluated_oracle_forecasts={evaluated_count}")
|
||||
|
||||
if args.summary:
|
||||
rows = store.load_evaluation_summary(
|
||||
lookback=timedelta(hours=args.lookback_hours)
|
||||
)
|
||||
print(f"oracle_evaluation_summary_rows={len(rows)}")
|
||||
for row in rows:
|
||||
print(
|
||||
" ".join(
|
||||
[
|
||||
f"kind={row['kind']}",
|
||||
f"model={row['model_version']}",
|
||||
f"horizon={row.get('horizon_label') or _format_horizon(row)}",
|
||||
f"n={row['evaluated_count']}",
|
||||
f"bias={_format_w(row['mean_error_w'])}",
|
||||
f"mae={_format_w(row['mean_absolute_error_w'])}",
|
||||
f"median_ae={_format_w(row['median_absolute_error_w'])}",
|
||||
f"mape={_format_pct(row['mean_absolute_pct_error'])}",
|
||||
f"coverage={_format_pct(row['interval_coverage'])}",
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Evaluate stored Astrape oracle predictions against Sigen actuals."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--evaluate",
|
||||
action="store_true",
|
||||
help="Evaluate stored predictions whose target time has passed.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--summary",
|
||||
action="store_true",
|
||||
help="Print evaluation quality by kind/model/horizon bucket.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--actual-window-minutes",
|
||||
type=float,
|
||||
default=5,
|
||||
help="Minutes after each target timestamp to average as realized actuals.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lookback-hours",
|
||||
type=float,
|
||||
default=168,
|
||||
help="Only evaluate/summarize predictions with target times this recent.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=1000,
|
||||
help="Maximum unevaluated predictions to process.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if not args.evaluate and not args.summary:
|
||||
args.evaluate = True
|
||||
args.summary = True
|
||||
return args
|
||||
|
||||
|
||||
def _format_w(value: object) -> str:
|
||||
if value is None:
|
||||
return "n/a"
|
||||
return f"{float(value):.0f}W"
|
||||
|
||||
|
||||
def _format_horizon(row: dict[str, object]) -> str:
|
||||
return f"{row['min_horizon_minutes']}-{row['max_horizon_minutes']}m"
|
||||
|
||||
|
||||
def _format_pct(value: object) -> str:
|
||||
if value is None:
|
||||
return "n/a"
|
||||
return f"{float(value) * 100:.1f}%"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,301 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from time import monotonic
|
||||
|
||||
from astrape_ems.classes.env import EnvLoader
|
||||
from astrape_ems.classes.store import SigenStore
|
||||
from astrape_oracle.classes.weather.store import WeatherStore
|
||||
from gibil.classes.usage_sequence_dataset import (
|
||||
UsageSequenceDatasetBuilder,
|
||||
UsageSequenceDatasetConfig,
|
||||
)
|
||||
from gibil.classes.usage_training import train_model
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
DEFAULT_MODEL_PATH = PROJECT_ROOT / "models" / "usage_sequence_tcn_v1.pt"
|
||||
DEFAULT_STATE_PATH = PROJECT_ROOT / "models" / "usage_sequence_training_state.json"
|
||||
DEFAULT_LOG_PATH = PROJECT_ROOT / "logs" / "usage_sequence_training.jsonl"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TrainingRegimeConfig:
|
||||
artifact_path: Path = DEFAULT_MODEL_PATH
|
||||
state_path: Path = DEFAULT_STATE_PATH
|
||||
log_path: Path = DEFAULT_LOG_PATH
|
||||
full_epochs: int = 20
|
||||
finetune_epochs: int = 1
|
||||
batch_size: int = 32
|
||||
full_learning_rate: float = 0.001
|
||||
finetune_learning_rate: float = 0.0002
|
||||
seed: int = 7
|
||||
threshold_seconds: float = 10 * 60
|
||||
terminal_threshold_seconds: float = 60 * 60
|
||||
nightly_interval_days: float = 1.0
|
||||
max_full_retrain_interval_days: float = 7.0
|
||||
target_finetune_seconds: float = 5 * 60
|
||||
finetune_min_samples: int = 32
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "TrainingRegimeConfig":
|
||||
EnvLoader().load()
|
||||
return cls(
|
||||
artifact_path=Path(
|
||||
os.environ.get(
|
||||
"ASTRAPE_USAGE_SEQUENCE_ARTIFACT_PATH",
|
||||
str(DEFAULT_MODEL_PATH),
|
||||
)
|
||||
),
|
||||
state_path=Path(
|
||||
os.environ.get(
|
||||
"ASTRAPE_USAGE_SEQUENCE_TRAINING_STATE_PATH",
|
||||
str(DEFAULT_STATE_PATH),
|
||||
)
|
||||
),
|
||||
log_path=Path(
|
||||
os.environ.get(
|
||||
"ASTRAPE_USAGE_SEQUENCE_TRAINING_LOG_PATH",
|
||||
str(DEFAULT_LOG_PATH),
|
||||
)
|
||||
),
|
||||
full_epochs=int(os.environ.get("ASTRAPE_USAGE_SEQUENCE_FULL_EPOCHS", "20")),
|
||||
finetune_epochs=int(
|
||||
os.environ.get("ASTRAPE_USAGE_SEQUENCE_FINETUNE_EPOCHS", "1")
|
||||
),
|
||||
batch_size=int(os.environ.get("ASTRAPE_USAGE_SEQUENCE_BATCH_SIZE", "32")),
|
||||
full_learning_rate=float(
|
||||
os.environ.get("ASTRAPE_USAGE_SEQUENCE_FULL_LR", "0.001")
|
||||
),
|
||||
finetune_learning_rate=float(
|
||||
os.environ.get("ASTRAPE_USAGE_SEQUENCE_FINETUNE_LR", "0.0002")
|
||||
),
|
||||
seed=int(os.environ.get("ASTRAPE_USAGE_SEQUENCE_SEED", "7")),
|
||||
threshold_seconds=float(
|
||||
os.environ.get("ASTRAPE_USAGE_SEQUENCE_FULL_T_SECONDS", str(10 * 60))
|
||||
),
|
||||
terminal_threshold_seconds=float(
|
||||
os.environ.get("ASTRAPE_USAGE_SEQUENCE_FULL_T2_SECONDS", str(60 * 60))
|
||||
),
|
||||
max_full_retrain_interval_days=float(
|
||||
os.environ.get("ASTRAPE_USAGE_SEQUENCE_MAX_FULL_INTERVAL_DAYS", "7")
|
||||
),
|
||||
target_finetune_seconds=float(
|
||||
os.environ.get("ASTRAPE_USAGE_SEQUENCE_TARGET_FINETUNE_SECONDS", str(5 * 60))
|
||||
),
|
||||
finetune_min_samples=int(
|
||||
os.environ.get("ASTRAPE_USAGE_SEQUENCE_FINETUNE_MIN_SAMPLES", "32")
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TrainingRegimeState:
|
||||
last_full_training_seconds: float | None = None
|
||||
last_full_sample_count: int | None = None
|
||||
last_full_completed_at: str | None = None
|
||||
last_finetune_completed_at: str | None = None
|
||||
full_retrain_locked: bool = False
|
||||
run_count: int = 0
|
||||
history: list[dict[str, object]] = field(default_factory=list)
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path) -> "TrainingRegimeState":
|
||||
if not path.exists():
|
||||
return cls()
|
||||
return cls(**json.loads(path.read_text()))
|
||||
|
||||
def save(self, path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(asdict(self), indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
class UsageSequenceTrainingScheduler:
|
||||
def __init__(self, config: TrainingRegimeConfig) -> None:
|
||||
self.config = config
|
||||
|
||||
def run(self, force_mode: str | None = None, dry_run: bool = False) -> dict[str, object]:
|
||||
state = TrainingRegimeState.load(self.config.state_path)
|
||||
decision = self._decision(state, force_mode=force_mode)
|
||||
if dry_run:
|
||||
event = {
|
||||
"created_at": self._now_iso(),
|
||||
"dry_run": True,
|
||||
"decision": decision,
|
||||
"state": asdict(state),
|
||||
}
|
||||
self._log_event(event)
|
||||
print(json.dumps(event, sort_keys=True))
|
||||
return event
|
||||
|
||||
dataset_config = UsageSequenceDatasetConfig.from_env()
|
||||
builder = UsageSequenceDatasetBuilder(
|
||||
config=dataset_config,
|
||||
plant_history=SigenStore.from_env(),
|
||||
weather_history=WeatherStore.from_env(),
|
||||
)
|
||||
examples = builder.build(limit=decision["sample_limit"])
|
||||
if not examples:
|
||||
raise SystemExit("No usage sequence examples available for scheduled training.")
|
||||
|
||||
artifact_path = self.config.artifact_path
|
||||
base_artifact_path = artifact_path if decision["mode"] == "finetune" else None
|
||||
if decision["mode"] == "finetune" and not artifact_path.exists():
|
||||
decision = self._full_decision(reason="missing_artifact")
|
||||
base_artifact_path = None
|
||||
|
||||
started = monotonic()
|
||||
result = train_model(
|
||||
examples=examples,
|
||||
builder=builder,
|
||||
epochs=decision["epochs"],
|
||||
batch_size=self.config.batch_size,
|
||||
learning_rate=decision["learning_rate"],
|
||||
artifact_path=artifact_path,
|
||||
seed=self.config.seed,
|
||||
base_artifact_path=base_artifact_path,
|
||||
mode=decision["mode"],
|
||||
)
|
||||
training_seconds = monotonic() - started
|
||||
|
||||
event = {
|
||||
"created_at": self._now_iso(),
|
||||
"mode": result.mode,
|
||||
"reason": decision["reason"],
|
||||
"training_seconds": training_seconds,
|
||||
"sample_count": result.sample_count,
|
||||
"training_examples": result.training_examples,
|
||||
"validation_examples": result.validation_examples,
|
||||
"final_train_loss": result.final_train_loss,
|
||||
"final_validation_loss": result.final_validation_loss,
|
||||
"artifact_path": str(result.artifact_path),
|
||||
"next_full_interval_days": self._full_interval_days(training_seconds),
|
||||
}
|
||||
self._update_state(state, event)
|
||||
state.save(self.config.state_path)
|
||||
self._log_event(event)
|
||||
print(json.dumps(event, sort_keys=True))
|
||||
return event
|
||||
|
||||
def _decision(
|
||||
self,
|
||||
state: TrainingRegimeState,
|
||||
force_mode: str | None,
|
||||
) -> dict[str, object]:
|
||||
if force_mode == "full":
|
||||
return self._full_decision(reason="forced_full")
|
||||
if force_mode == "finetune":
|
||||
return self._finetune_decision(state, reason="forced_finetune")
|
||||
if state.full_retrain_locked:
|
||||
return self._finetune_decision(state, reason="full_retrain_locked")
|
||||
if state.last_full_training_seconds is None:
|
||||
return self._full_decision(reason="no_full_training_history")
|
||||
if self._due_for_full_retrain(state):
|
||||
return self._full_decision(reason="full_retrain_due")
|
||||
return self._finetune_decision(state, reason="nightly_finetune")
|
||||
|
||||
def _due_for_full_retrain(self, state: TrainingRegimeState) -> bool:
|
||||
if state.last_full_completed_at is None:
|
||||
return True
|
||||
completed_at = datetime.fromisoformat(state.last_full_completed_at)
|
||||
due_after = completed_at + timedelta(
|
||||
days=self._full_interval_days(state.last_full_training_seconds)
|
||||
)
|
||||
return datetime.now(timezone.utc) >= due_after
|
||||
|
||||
def _full_decision(self, reason: str) -> dict[str, object]:
|
||||
return {
|
||||
"mode": "full",
|
||||
"reason": reason,
|
||||
"sample_limit": None,
|
||||
"epochs": self.config.full_epochs,
|
||||
"learning_rate": self.config.full_learning_rate,
|
||||
}
|
||||
|
||||
def _finetune_decision(
|
||||
self,
|
||||
state: TrainingRegimeState,
|
||||
reason: str,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"mode": "finetune",
|
||||
"reason": reason,
|
||||
"sample_limit": self._finetune_sample_limit(state),
|
||||
"epochs": self.config.finetune_epochs,
|
||||
"learning_rate": self.config.finetune_learning_rate,
|
||||
}
|
||||
|
||||
def _finetune_sample_limit(self, state: TrainingRegimeState) -> int | None:
|
||||
if not state.last_full_training_seconds or not state.last_full_sample_count:
|
||||
return None
|
||||
samples_per_second = state.last_full_sample_count / state.last_full_training_seconds
|
||||
sample_limit = round(samples_per_second * self.config.target_finetune_seconds)
|
||||
return max(self.config.finetune_min_samples, sample_limit)
|
||||
|
||||
def _full_interval_days(self, training_seconds: float | None) -> float:
|
||||
if training_seconds is None or training_seconds <= self.config.threshold_seconds:
|
||||
return self.config.nightly_interval_days
|
||||
if training_seconds >= self.config.terminal_threshold_seconds:
|
||||
return self.config.max_full_retrain_interval_days
|
||||
span = self.config.terminal_threshold_seconds - self.config.threshold_seconds
|
||||
progress = (training_seconds - self.config.threshold_seconds) / span
|
||||
return self.config.nightly_interval_days + progress * (
|
||||
self.config.max_full_retrain_interval_days - self.config.nightly_interval_days
|
||||
)
|
||||
|
||||
def _update_state(
|
||||
self,
|
||||
state: TrainingRegimeState,
|
||||
event: dict[str, object],
|
||||
) -> None:
|
||||
state.run_count += 1
|
||||
if event["mode"] == "full":
|
||||
state.last_full_training_seconds = float(event["training_seconds"])
|
||||
state.last_full_sample_count = int(event["sample_count"])
|
||||
state.last_full_completed_at = str(event["created_at"])
|
||||
if state.last_full_training_seconds >= self.config.terminal_threshold_seconds:
|
||||
state.full_retrain_locked = True
|
||||
else:
|
||||
state.last_finetune_completed_at = str(event["created_at"])
|
||||
state.history.append(event)
|
||||
state.history = state.history[-30:]
|
||||
|
||||
def _log_event(self, event: dict[str, object]) -> None:
|
||||
self.config.log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self.config.log_path.open("a") as handle:
|
||||
handle.write(json.dumps(event, sort_keys=True) + "\n")
|
||||
|
||||
def _now_iso(self) -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Adaptive full-retrain/fine-tune scheduler for the usage TCN."
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument(
|
||||
"--force-mode",
|
||||
choices=("full", "finetune"),
|
||||
default=None,
|
||||
help="Override the adaptive decision for this run.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
EnvLoader().load()
|
||||
args = parse_args()
|
||||
UsageSequenceTrainingScheduler(TrainingRegimeConfig.from_env()).run(
|
||||
force_mode=args.force_mode,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,158 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from os import environ
|
||||
from pathlib import Path
|
||||
|
||||
from astrape_ems.classes.env import EnvLoader
|
||||
from astrape_ems.classes.store import SigenStore
|
||||
from astrape_oracle.classes.weather.store import WeatherStore
|
||||
from gibil.classes.usage_sequence_dataset import (
|
||||
UsageSequenceDatasetBuilder,
|
||||
UsageSequenceDatasetConfig,
|
||||
)
|
||||
from gibil.classes.usage_training import train_model
|
||||
|
||||
|
||||
def main() -> None:
|
||||
EnvLoader().load()
|
||||
environ.setdefault("TORCH_CPP_LOG_LEVEL", "ERROR")
|
||||
args = parse_args()
|
||||
config = UsageSequenceDatasetConfig.from_env()
|
||||
builder = UsageSequenceDatasetBuilder(
|
||||
config=config,
|
||||
plant_history=SigenStore.from_env(),
|
||||
weather_history=WeatherStore.from_env(),
|
||||
)
|
||||
examples = builder.build(limit=args.limit)
|
||||
|
||||
print(f"usage_sequence_examples={len(examples)}")
|
||||
print(
|
||||
"minimum_history_hours="
|
||||
f"{builder.max_past_hours + config.future_hours}"
|
||||
)
|
||||
print(f"past_features={len(builder.past_feature_names)}")
|
||||
for scale in config.past_scales:
|
||||
print(
|
||||
f"past_scale={scale.name} "
|
||||
f"hours={scale.hours} "
|
||||
f"step_seconds={scale.step_seconds} "
|
||||
f"steps={builder.past_steps(scale)}"
|
||||
)
|
||||
print(f"future_steps={builder.future_steps}")
|
||||
print(f"future_features={len(builder.future_feature_names)}")
|
||||
|
||||
if examples:
|
||||
first = examples[0]
|
||||
last = examples[-1]
|
||||
print(f"first_issued_at={first.issued_at.isoformat()}")
|
||||
print(f"last_issued_at={last.issued_at.isoformat()}")
|
||||
for name, rows in first.past_by_scale.items():
|
||||
print(f"first_past_{name}_shape={len(rows)}x{len(rows[0])}")
|
||||
token_count = sum(
|
||||
len(tokens)
|
||||
for tokens in first.past_tokens_by_scale[name]
|
||||
)
|
||||
print(f"first_past_{name}_tokens={token_count}")
|
||||
print(
|
||||
"first_future_feature_shape="
|
||||
f"{len(first.future_features)}x{len(first.future_features[0])}"
|
||||
)
|
||||
print(
|
||||
"first_future_tokens="
|
||||
f"{sum(len(tokens) for tokens in first.future_tokens)}"
|
||||
)
|
||||
print(f"first_targets={len(first.targets)}")
|
||||
print(
|
||||
"first_target_preview="
|
||||
+ ",".join(f"{value:.0f}" for value in first.targets[:8])
|
||||
)
|
||||
|
||||
if args.dry_run:
|
||||
return
|
||||
|
||||
if not examples:
|
||||
raise SystemExit("No usage sequence examples available for training yet.")
|
||||
|
||||
result = train_model(
|
||||
examples=examples,
|
||||
builder=builder,
|
||||
epochs=args.epochs,
|
||||
batch_size=args.batch_size,
|
||||
learning_rate=args.learning_rate,
|
||||
artifact_path=args.artifact_path,
|
||||
seed=args.seed,
|
||||
base_artifact_path=args.base_artifact_path,
|
||||
mode=args.mode,
|
||||
)
|
||||
print(f"training_mode={result.mode}")
|
||||
print(f"training_examples={result.training_examples}")
|
||||
print(f"validation_examples={result.validation_examples}")
|
||||
if result.final_train_loss is not None:
|
||||
print(f"final_train_pinball_loss={result.final_train_loss:.4f}")
|
||||
if result.final_validation_loss is not None:
|
||||
print(f"final_validation_pinball_loss={result.final_validation_loss:.4f}")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Build training windows for the sequence usage oracle."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Only build examples and print dataset shape.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Optional maximum number of examples to build.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--epochs",
|
||||
type=int,
|
||||
default=20,
|
||||
help="Training epochs.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-size",
|
||||
type=int,
|
||||
default=32,
|
||||
help="Training batch size.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--learning-rate",
|
||||
type=float,
|
||||
default=0.001,
|
||||
help="Adam learning rate.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--artifact-path",
|
||||
type=Path,
|
||||
default=Path("models/usage_sequence_tcn_v1.pt"),
|
||||
help="Where to save the trained TCN artifact.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--base-artifact-path",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Existing artifact to continue training from.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
choices=("full", "finetune"),
|
||||
default="full",
|
||||
help="Training mode recorded in the saved artifact.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=7,
|
||||
help="Deterministic shuffle seed.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user