Flatten to a single package directory
The checkout is now astrape/astrape-oracle, not astrape/astrape-oracle/<package>. The package contents move to the repository root and pyproject maps the package name to '.' via package-dir, with the subpackages listed explicitly. The systemd units move out to /srv/astrape/deploy/systemd, outside every repository, because they describe this deployment rather than any one component. Verified both install modes: editable, which is how the deployment runs, and a plain wheel install, which is what the container build does. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
rpotter6298
@@ -0,0 +1,122 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
from os import environ
|
||||
from sys import stderr
|
||||
from time import sleep
|
||||
|
||||
from gibil.classes.oracle_builder import EnergyOracleBuilder
|
||||
from gibil.classes.forecast_config import EnergyForecastConfig
|
||||
from astrape_ems.classes.env import EnvLoader
|
||||
from astrape_oracle.classes.oracle_store import OracleStore
|
||||
from astrape_ems.classes.store import SigenStore
|
||||
from astrape_oracle.classes.weather.store import WeatherStore
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OracleDaemonConfig:
|
||||
poll_seconds: float
|
||||
evaluate_forecasts: bool
|
||||
evaluation_actual_window_minutes: float
|
||||
evaluation_lookback_hours: float
|
||||
evaluation_limit: int
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "OracleDaemonConfig":
|
||||
return cls(
|
||||
poll_seconds=float(environ.get("ASTRAPE_ORACLE_POLL_SECONDS", "300")),
|
||||
evaluate_forecasts=environ.get(
|
||||
"ASTRAPE_ORACLE_EVALUATE_FORECASTS", "1"
|
||||
).lower()
|
||||
not in {"0", "false", "no"},
|
||||
evaluation_actual_window_minutes=float(
|
||||
environ.get("ASTRAPE_ORACLE_EVALUATION_WINDOW_MINUTES", "5")
|
||||
),
|
||||
evaluation_lookback_hours=float(
|
||||
environ.get("ASTRAPE_ORACLE_EVALUATION_LOOKBACK_HOURS", "168")
|
||||
),
|
||||
evaluation_limit=int(environ.get("ASTRAPE_ORACLE_EVALUATION_LIMIT", "1000")),
|
||||
)
|
||||
|
||||
|
||||
class OracleDaemon:
|
||||
"""Periodically stores oracle projection curves for evaluation."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: OracleDaemonConfig,
|
||||
builder: EnergyOracleBuilder,
|
||||
store: OracleStore,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.builder = builder
|
||||
self.store = store
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "OracleDaemon":
|
||||
return cls(
|
||||
config=OracleDaemonConfig.from_env(),
|
||||
builder=EnergyOracleBuilder(
|
||||
weather_history=WeatherStore.from_env(),
|
||||
plant_history=SigenStore.from_env(),
|
||||
config=EnergyForecastConfig.from_env(),
|
||||
),
|
||||
store=OracleStore.from_env(),
|
||||
)
|
||||
|
||||
def run_once(self) -> int:
|
||||
solar_run, load_run, net_run = self.builder.build()
|
||||
saved_count = self.store.save_runs(solar_run, load_run, net_run)
|
||||
if self.config.evaluate_forecasts:
|
||||
from datetime import timedelta
|
||||
|
||||
evaluated_count = self.store.evaluate_due_forecasts(
|
||||
actual_window=timedelta(
|
||||
minutes=self.config.evaluation_actual_window_minutes
|
||||
),
|
||||
lookback=timedelta(hours=self.config.evaluation_lookback_hours),
|
||||
limit=self.config.evaluation_limit,
|
||||
)
|
||||
return saved_count + evaluated_count
|
||||
return saved_count
|
||||
|
||||
def run_forever(self) -> None:
|
||||
self.store.initialize()
|
||||
while True:
|
||||
try:
|
||||
saved_count = self.run_once()
|
||||
print(f"stored_oracle_records={saved_count}", flush=True)
|
||||
except Exception as error:
|
||||
print(f"oracle_poll_error={error}", file=stderr, flush=True)
|
||||
sleep(self.config.poll_seconds)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
EnvLoader().load()
|
||||
args = parse_args()
|
||||
daemon = OracleDaemon.from_env()
|
||||
if args.once:
|
||||
print(f"stored_oracle_records={daemon.run_once()}", flush=True)
|
||||
return
|
||||
daemon.run_forever()
|
||||
except Exception as error:
|
||||
print(f"oracle_daemon_startup_error={error}", file=stderr)
|
||||
raise SystemExit(1) from error
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Store Astrape oracle projection curves."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--once",
|
||||
action="store_true",
|
||||
help="Save one set of oracle curves and exit.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,137 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from os import environ
|
||||
from sys import stderr
|
||||
from time import sleep
|
||||
|
||||
from astrape_ems.classes.env import EnvLoader
|
||||
from astrape_oracle.classes.weather.openmeteo import (
|
||||
OpenMeteoArchiveClient,
|
||||
OpenMeteoClient,
|
||||
WeatherBuilder,
|
||||
)
|
||||
from astrape_oracle.classes.weather.store import WeatherStore
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DbDaemonConfig:
|
||||
latitude: float
|
||||
longitude: float
|
||||
forecast_hours: int
|
||||
truth_lookback_days: int
|
||||
truth_end_delay_days: int
|
||||
poll_seconds: int
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "DbDaemonConfig":
|
||||
return cls(
|
||||
latitude=float(_required_env("ASTRAPE_LATITUDE")),
|
||||
longitude=float(_required_env("ASTRAPE_LONGITUDE")),
|
||||
forecast_hours=int(environ.get("ASTRAPE_WEATHER_FORECAST_HOURS", "48")),
|
||||
truth_lookback_days=int(
|
||||
environ.get("ASTRAPE_WEATHER_TRUTH_LOOKBACK_DAYS", "14")
|
||||
),
|
||||
truth_end_delay_days=int(
|
||||
environ.get("ASTRAPE_WEATHER_TRUTH_END_DELAY_DAYS", "5")
|
||||
),
|
||||
poll_seconds=int(environ.get("ASTRAPE_WEATHER_POLL_SECONDS", "3600")),
|
||||
)
|
||||
|
||||
|
||||
class DbDaemon:
|
||||
"""Runs builder components that populate Astrape's database."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: DbDaemonConfig,
|
||||
weather_client: OpenMeteoClient,
|
||||
archive_client: OpenMeteoArchiveClient,
|
||||
weather_builder: WeatherBuilder,
|
||||
weather_store: WeatherStore,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.weather_client = weather_client
|
||||
self.archive_client = archive_client
|
||||
self.weather_builder = weather_builder
|
||||
self.weather_store = weather_store
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "DbDaemon":
|
||||
return cls(
|
||||
config=DbDaemonConfig.from_env(),
|
||||
weather_client=OpenMeteoClient(),
|
||||
archive_client=OpenMeteoArchiveClient(),
|
||||
weather_builder=WeatherBuilder(),
|
||||
weather_store=WeatherStore.from_env(),
|
||||
)
|
||||
|
||||
def initialize(self) -> None:
|
||||
self.weather_store.initialize()
|
||||
|
||||
def run_once(self) -> tuple[int, int]:
|
||||
raw_run = self.weather_client.fetch_forecast(
|
||||
latitude=self.config.latitude,
|
||||
longitude=self.config.longitude,
|
||||
forecast_hours=self.config.forecast_hours,
|
||||
)
|
||||
forecast_run = self.weather_builder.build_forecast_run(
|
||||
source=raw_run.source,
|
||||
latitude=raw_run.latitude,
|
||||
longitude=raw_run.longitude,
|
||||
points=raw_run.points,
|
||||
issued_at=raw_run.issued_at,
|
||||
)
|
||||
forecast_count = self.weather_store.save_forecast_run(forecast_run)
|
||||
zero_hour_truth_count = self.weather_store.save_zero_hour_forecast_as_truth(
|
||||
forecast_run
|
||||
)
|
||||
|
||||
today = datetime.now(timezone.utc).date()
|
||||
truth_end = today - timedelta(days=self.config.truth_end_delay_days)
|
||||
truth_start = truth_end - timedelta(days=self.config.truth_lookback_days)
|
||||
truth_points = self.archive_client.fetch_resolved_truth(
|
||||
latitude=self.config.latitude,
|
||||
longitude=self.config.longitude,
|
||||
start_date=truth_start,
|
||||
end_date=truth_end,
|
||||
)
|
||||
archive_truth_count = self.weather_store.save_resolved_truth(truth_points)
|
||||
|
||||
return forecast_count, zero_hour_truth_count + archive_truth_count
|
||||
|
||||
def run_forever(self) -> None:
|
||||
self.initialize()
|
||||
while True:
|
||||
forecast_count, truth_count = self.run_once()
|
||||
print(
|
||||
f"stored_weather_forecast_points={forecast_count} "
|
||||
f"stored_weather_resolved_truth={truth_count}",
|
||||
flush=True,
|
||||
)
|
||||
sleep(self.config.poll_seconds)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
EnvLoader().load()
|
||||
daemon = DbDaemon.from_env()
|
||||
daemon.run_forever()
|
||||
except Exception as error:
|
||||
print(f"db_daemon_startup_error={error}", file=stderr)
|
||||
raise SystemExit(1) from error
|
||||
|
||||
|
||||
def _required_env(name: str) -> str:
|
||||
value = environ.get(name)
|
||||
if not value:
|
||||
raise RuntimeError(
|
||||
f"{name} is required. Set ASTRAPE_DATABASE_URL, "
|
||||
"ASTRAPE_LATITUDE, and ASTRAPE_LONGITUDE before starting db_daemon."
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,113 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
from os import environ
|
||||
from sys import stderr
|
||||
from time import sleep
|
||||
|
||||
from astrape_ems.classes.env import EnvLoader
|
||||
from astrape_oracle.classes.weather.ecowitt import EcowittGw3000Client
|
||||
from astrape_oracle.classes.weather.store import WeatherStore
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WeatherStationDaemonConfig:
|
||||
host: str
|
||||
poll_seconds: float
|
||||
timeout_seconds: float
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "WeatherStationDaemonConfig":
|
||||
host = environ.get("ECOWITT_GW3000_HOST")
|
||||
if not host:
|
||||
raise RuntimeError("ECOWITT_GW3000_HOST is required")
|
||||
|
||||
return cls(
|
||||
host=host,
|
||||
poll_seconds=float(environ.get("ECOWITT_GW3000_POLL_SECONDS", "60")),
|
||||
timeout_seconds=float(environ.get("ECOWITT_GW3000_TIMEOUT_SECONDS", "5")),
|
||||
)
|
||||
|
||||
|
||||
class WeatherStationDaemon:
|
||||
"""Polls local weather station observations into TimescaleDB."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WeatherStationDaemonConfig,
|
||||
client: EcowittGw3000Client,
|
||||
weather_store: WeatherStore,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.client = client
|
||||
self.weather_store = weather_store
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "WeatherStationDaemon":
|
||||
config = WeatherStationDaemonConfig.from_env()
|
||||
return cls(
|
||||
config=config,
|
||||
client=EcowittGw3000Client(
|
||||
host=config.host,
|
||||
timeout_seconds=config.timeout_seconds,
|
||||
),
|
||||
weather_store=WeatherStore.from_env(),
|
||||
)
|
||||
|
||||
def initialize(self) -> None:
|
||||
self.weather_store.initialize()
|
||||
|
||||
def run_once(self) -> int:
|
||||
observation = self.client.fetch_observation()
|
||||
return self.weather_store.save_station_observation(observation)
|
||||
|
||||
def run_forever(self) -> None:
|
||||
self.initialize()
|
||||
while True:
|
||||
try:
|
||||
saved_count = self.run_once()
|
||||
print(
|
||||
f"stored_weather_station_observations={saved_count}",
|
||||
flush=True,
|
||||
)
|
||||
except Exception as error:
|
||||
print(f"weather_station_poll_error={error}", file=stderr, flush=True)
|
||||
|
||||
sleep(self.config.poll_seconds)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
EnvLoader().load()
|
||||
daemon = WeatherStationDaemon.from_env()
|
||||
args = parse_args()
|
||||
if args.once:
|
||||
daemon.initialize()
|
||||
saved_count = daemon.run_once()
|
||||
print(
|
||||
f"stored_weather_station_observations={saved_count}",
|
||||
flush=True,
|
||||
)
|
||||
return
|
||||
|
||||
daemon.run_forever()
|
||||
except Exception as error:
|
||||
print(f"weather_station_daemon_startup_error={error}", file=stderr)
|
||||
raise SystemExit(1) from error
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Poll local Ecowitt weather station observations into Astrape."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--once",
|
||||
action="store_true",
|
||||
help="Initialize storage, save one observation, and exit.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Debug script to find duplicate target times in forecast data."""
|
||||
|
||||
from astrape_ems.classes.env import EnvLoader
|
||||
from astrape_oracle.classes.weather.store import WeatherStore
|
||||
from collections import defaultdict
|
||||
|
||||
EnvLoader().load()
|
||||
|
||||
store = WeatherStore.from_env()
|
||||
dataset = store.load_display_dataset()
|
||||
|
||||
# Group by (target_at, horizon_hours) to find duplicates
|
||||
by_key = defaultdict(list)
|
||||
for point in dataset.forecast_points:
|
||||
key = (point.target_at, point.horizon_hours)
|
||||
by_key[key].append(point)
|
||||
|
||||
# Find duplicates
|
||||
duplicates = {k: v for k, v in by_key.items() if len(v) > 1}
|
||||
|
||||
print(f"\nTotal forecast points: {len(dataset.forecast_points)}")
|
||||
print(f"Unique (target_at, horizon) pairs: {len(by_key)}")
|
||||
print(f"Duplicate (target_at, horizon) pairs: {len(duplicates)}")
|
||||
|
||||
if duplicates:
|
||||
print("\nFirst 3 duplicates:")
|
||||
for (target_at, horizon), points in list(duplicates.items())[:3]:
|
||||
print(f"\n target_at={target_at}, horizon={horizon}h ({len(points)} points):")
|
||||
for i, p in enumerate(points):
|
||||
print(f" [{i}] issued_at={p.issued_at}, temp={p.temperature_c}, source={p.source}")
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Debug baseline energy forecast curves."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import timezone
|
||||
|
||||
from gibil.classes.oracle_builder import EnergyForecastBuilder
|
||||
from gibil.classes.forecast_config import EnergyForecastConfig
|
||||
from astrape_ems.classes.env import EnvLoader
|
||||
from astrape_ems.classes.store import SigenStore
|
||||
from astrape_oracle.classes.weather.store import WeatherStore
|
||||
from gibil.classes.models import PowerForecastPoint
|
||||
|
||||
|
||||
def main() -> None:
|
||||
EnvLoader().load()
|
||||
args = parse_args()
|
||||
solar_run, load_run, net_run = EnergyForecastBuilder(
|
||||
weather_history=WeatherStore.from_env(),
|
||||
plant_history=SigenStore.from_env(),
|
||||
config=EnergyForecastConfig.from_env(),
|
||||
).build()
|
||||
|
||||
print(
|
||||
f"issued_at={net_run.issued_at.astimezone(timezone.utc).isoformat(timespec='seconds')}"
|
||||
)
|
||||
print(
|
||||
f"solar_model={solar_run.model_version} "
|
||||
f"load_model={load_run.model_version} points={len(net_run.points)}"
|
||||
)
|
||||
print(
|
||||
"target_at solar_p10 solar_p50 solar_p90 "
|
||||
"load_p10 load_p50 load_p90 net_p10 net_p50 net_p90"
|
||||
)
|
||||
solar_by_target = _by_target(solar_run.points)
|
||||
load_by_target = _by_target(load_run.points)
|
||||
for point in net_run.points[: args.limit]:
|
||||
solar_point = solar_by_target[point.target_at]
|
||||
load_point = load_by_target[point.target_at]
|
||||
print(
|
||||
f"{point.target_at.astimezone(timezone.utc).isoformat(timespec='minutes'):25} "
|
||||
f"{solar_point.p10_power_w:9.0f} "
|
||||
f"{solar_point.p50_power_w:9.0f} "
|
||||
f"{solar_point.p90_power_w:9.0f} "
|
||||
f"{load_point.p10_power_w:8.0f} "
|
||||
f"{load_point.p50_power_w:8.0f} "
|
||||
f"{load_point.p90_power_w:8.0f} "
|
||||
f"{point.p10_net_power_w:7.0f} "
|
||||
f"{point.p50_net_power_w:7.0f} "
|
||||
f"{point.p90_net_power_w:7.0f}"
|
||||
)
|
||||
|
||||
|
||||
def _by_target(points: list[PowerForecastPoint]) -> dict[object, PowerForecastPoint]:
|
||||
return {point.target_at: point for point in points}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Print baseline solar/load/net forecast curves."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=24,
|
||||
help="Number of forecast points to show. Defaults to 24.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,5 @@
|
||||
from astrape_oracle.scripts.evaluator import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,102 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import timedelta
|
||||
|
||||
from astrape_ems.classes.env import EnvLoader
|
||||
from astrape_oracle.classes.oracle_store import OracleStore
|
||||
|
||||
|
||||
def main() -> None:
|
||||
EnvLoader().load()
|
||||
args = parse_args()
|
||||
store = OracleStore.from_env()
|
||||
|
||||
if args.evaluate:
|
||||
evaluated_count = store.evaluate_due_forecasts(
|
||||
actual_window=timedelta(minutes=args.actual_window_minutes),
|
||||
lookback=timedelta(hours=args.lookback_hours),
|
||||
limit=args.limit,
|
||||
)
|
||||
print(f"evaluated_oracle_forecasts={evaluated_count}")
|
||||
|
||||
if args.summary:
|
||||
rows = store.load_evaluation_summary(
|
||||
lookback=timedelta(hours=args.lookback_hours)
|
||||
)
|
||||
print(f"oracle_evaluation_summary_rows={len(rows)}")
|
||||
for row in rows:
|
||||
print(
|
||||
" ".join(
|
||||
[
|
||||
f"kind={row['kind']}",
|
||||
f"model={row['model_version']}",
|
||||
f"horizon={row.get('horizon_label') or _format_horizon(row)}",
|
||||
f"n={row['evaluated_count']}",
|
||||
f"bias={_format_w(row['mean_error_w'])}",
|
||||
f"mae={_format_w(row['mean_absolute_error_w'])}",
|
||||
f"median_ae={_format_w(row['median_absolute_error_w'])}",
|
||||
f"mape={_format_pct(row['mean_absolute_pct_error'])}",
|
||||
f"coverage={_format_pct(row['interval_coverage'])}",
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Evaluate stored Astrape oracle predictions against Sigen actuals."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--evaluate",
|
||||
action="store_true",
|
||||
help="Evaluate stored predictions whose target time has passed.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--summary",
|
||||
action="store_true",
|
||||
help="Print evaluation quality by kind/model/horizon bucket.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--actual-window-minutes",
|
||||
type=float,
|
||||
default=5,
|
||||
help="Minutes after each target timestamp to average as realized actuals.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lookback-hours",
|
||||
type=float,
|
||||
default=168,
|
||||
help="Only evaluate/summarize predictions with target times this recent.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=1000,
|
||||
help="Maximum unevaluated predictions to process.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if not args.evaluate and not args.summary:
|
||||
args.evaluate = True
|
||||
args.summary = True
|
||||
return args
|
||||
|
||||
|
||||
def _format_w(value: object) -> str:
|
||||
if value is None:
|
||||
return "n/a"
|
||||
return f"{float(value):.0f}W"
|
||||
|
||||
|
||||
def _format_horizon(row: dict[str, object]) -> str:
|
||||
return f"{row['min_horizon_minutes']}-{row['max_horizon_minutes']}m"
|
||||
|
||||
|
||||
def _format_pct(value: object) -> str:
|
||||
if value is None:
|
||||
return "n/a"
|
||||
return f"{float(value) * 100:.1f}%"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,301 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from time import monotonic
|
||||
|
||||
from astrape_ems.classes.env import EnvLoader
|
||||
from astrape_ems.classes.store import SigenStore
|
||||
from astrape_oracle.classes.weather.store import WeatherStore
|
||||
from gibil.classes.usage_sequence_dataset import (
|
||||
UsageSequenceDatasetBuilder,
|
||||
UsageSequenceDatasetConfig,
|
||||
)
|
||||
from gibil.classes.usage_training import train_model
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
DEFAULT_MODEL_PATH = PROJECT_ROOT / "models" / "usage_sequence_tcn_v1.pt"
|
||||
DEFAULT_STATE_PATH = PROJECT_ROOT / "models" / "usage_sequence_training_state.json"
|
||||
DEFAULT_LOG_PATH = PROJECT_ROOT / "logs" / "usage_sequence_training.jsonl"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TrainingRegimeConfig:
|
||||
artifact_path: Path = DEFAULT_MODEL_PATH
|
||||
state_path: Path = DEFAULT_STATE_PATH
|
||||
log_path: Path = DEFAULT_LOG_PATH
|
||||
full_epochs: int = 20
|
||||
finetune_epochs: int = 1
|
||||
batch_size: int = 32
|
||||
full_learning_rate: float = 0.001
|
||||
finetune_learning_rate: float = 0.0002
|
||||
seed: int = 7
|
||||
threshold_seconds: float = 10 * 60
|
||||
terminal_threshold_seconds: float = 60 * 60
|
||||
nightly_interval_days: float = 1.0
|
||||
max_full_retrain_interval_days: float = 7.0
|
||||
target_finetune_seconds: float = 5 * 60
|
||||
finetune_min_samples: int = 32
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "TrainingRegimeConfig":
|
||||
EnvLoader().load()
|
||||
return cls(
|
||||
artifact_path=Path(
|
||||
os.environ.get(
|
||||
"ASTRAPE_USAGE_SEQUENCE_ARTIFACT_PATH",
|
||||
str(DEFAULT_MODEL_PATH),
|
||||
)
|
||||
),
|
||||
state_path=Path(
|
||||
os.environ.get(
|
||||
"ASTRAPE_USAGE_SEQUENCE_TRAINING_STATE_PATH",
|
||||
str(DEFAULT_STATE_PATH),
|
||||
)
|
||||
),
|
||||
log_path=Path(
|
||||
os.environ.get(
|
||||
"ASTRAPE_USAGE_SEQUENCE_TRAINING_LOG_PATH",
|
||||
str(DEFAULT_LOG_PATH),
|
||||
)
|
||||
),
|
||||
full_epochs=int(os.environ.get("ASTRAPE_USAGE_SEQUENCE_FULL_EPOCHS", "20")),
|
||||
finetune_epochs=int(
|
||||
os.environ.get("ASTRAPE_USAGE_SEQUENCE_FINETUNE_EPOCHS", "1")
|
||||
),
|
||||
batch_size=int(os.environ.get("ASTRAPE_USAGE_SEQUENCE_BATCH_SIZE", "32")),
|
||||
full_learning_rate=float(
|
||||
os.environ.get("ASTRAPE_USAGE_SEQUENCE_FULL_LR", "0.001")
|
||||
),
|
||||
finetune_learning_rate=float(
|
||||
os.environ.get("ASTRAPE_USAGE_SEQUENCE_FINETUNE_LR", "0.0002")
|
||||
),
|
||||
seed=int(os.environ.get("ASTRAPE_USAGE_SEQUENCE_SEED", "7")),
|
||||
threshold_seconds=float(
|
||||
os.environ.get("ASTRAPE_USAGE_SEQUENCE_FULL_T_SECONDS", str(10 * 60))
|
||||
),
|
||||
terminal_threshold_seconds=float(
|
||||
os.environ.get("ASTRAPE_USAGE_SEQUENCE_FULL_T2_SECONDS", str(60 * 60))
|
||||
),
|
||||
max_full_retrain_interval_days=float(
|
||||
os.environ.get("ASTRAPE_USAGE_SEQUENCE_MAX_FULL_INTERVAL_DAYS", "7")
|
||||
),
|
||||
target_finetune_seconds=float(
|
||||
os.environ.get("ASTRAPE_USAGE_SEQUENCE_TARGET_FINETUNE_SECONDS", str(5 * 60))
|
||||
),
|
||||
finetune_min_samples=int(
|
||||
os.environ.get("ASTRAPE_USAGE_SEQUENCE_FINETUNE_MIN_SAMPLES", "32")
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TrainingRegimeState:
|
||||
last_full_training_seconds: float | None = None
|
||||
last_full_sample_count: int | None = None
|
||||
last_full_completed_at: str | None = None
|
||||
last_finetune_completed_at: str | None = None
|
||||
full_retrain_locked: bool = False
|
||||
run_count: int = 0
|
||||
history: list[dict[str, object]] = field(default_factory=list)
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path) -> "TrainingRegimeState":
|
||||
if not path.exists():
|
||||
return cls()
|
||||
return cls(**json.loads(path.read_text()))
|
||||
|
||||
def save(self, path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(asdict(self), indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
class UsageSequenceTrainingScheduler:
|
||||
def __init__(self, config: TrainingRegimeConfig) -> None:
|
||||
self.config = config
|
||||
|
||||
def run(self, force_mode: str | None = None, dry_run: bool = False) -> dict[str, object]:
|
||||
state = TrainingRegimeState.load(self.config.state_path)
|
||||
decision = self._decision(state, force_mode=force_mode)
|
||||
if dry_run:
|
||||
event = {
|
||||
"created_at": self._now_iso(),
|
||||
"dry_run": True,
|
||||
"decision": decision,
|
||||
"state": asdict(state),
|
||||
}
|
||||
self._log_event(event)
|
||||
print(json.dumps(event, sort_keys=True))
|
||||
return event
|
||||
|
||||
dataset_config = UsageSequenceDatasetConfig.from_env()
|
||||
builder = UsageSequenceDatasetBuilder(
|
||||
config=dataset_config,
|
||||
plant_history=SigenStore.from_env(),
|
||||
weather_history=WeatherStore.from_env(),
|
||||
)
|
||||
examples = builder.build(limit=decision["sample_limit"])
|
||||
if not examples:
|
||||
raise SystemExit("No usage sequence examples available for scheduled training.")
|
||||
|
||||
artifact_path = self.config.artifact_path
|
||||
base_artifact_path = artifact_path if decision["mode"] == "finetune" else None
|
||||
if decision["mode"] == "finetune" and not artifact_path.exists():
|
||||
decision = self._full_decision(reason="missing_artifact")
|
||||
base_artifact_path = None
|
||||
|
||||
started = monotonic()
|
||||
result = train_model(
|
||||
examples=examples,
|
||||
builder=builder,
|
||||
epochs=decision["epochs"],
|
||||
batch_size=self.config.batch_size,
|
||||
learning_rate=decision["learning_rate"],
|
||||
artifact_path=artifact_path,
|
||||
seed=self.config.seed,
|
||||
base_artifact_path=base_artifact_path,
|
||||
mode=decision["mode"],
|
||||
)
|
||||
training_seconds = monotonic() - started
|
||||
|
||||
event = {
|
||||
"created_at": self._now_iso(),
|
||||
"mode": result.mode,
|
||||
"reason": decision["reason"],
|
||||
"training_seconds": training_seconds,
|
||||
"sample_count": result.sample_count,
|
||||
"training_examples": result.training_examples,
|
||||
"validation_examples": result.validation_examples,
|
||||
"final_train_loss": result.final_train_loss,
|
||||
"final_validation_loss": result.final_validation_loss,
|
||||
"artifact_path": str(result.artifact_path),
|
||||
"next_full_interval_days": self._full_interval_days(training_seconds),
|
||||
}
|
||||
self._update_state(state, event)
|
||||
state.save(self.config.state_path)
|
||||
self._log_event(event)
|
||||
print(json.dumps(event, sort_keys=True))
|
||||
return event
|
||||
|
||||
def _decision(
|
||||
self,
|
||||
state: TrainingRegimeState,
|
||||
force_mode: str | None,
|
||||
) -> dict[str, object]:
|
||||
if force_mode == "full":
|
||||
return self._full_decision(reason="forced_full")
|
||||
if force_mode == "finetune":
|
||||
return self._finetune_decision(state, reason="forced_finetune")
|
||||
if state.full_retrain_locked:
|
||||
return self._finetune_decision(state, reason="full_retrain_locked")
|
||||
if state.last_full_training_seconds is None:
|
||||
return self._full_decision(reason="no_full_training_history")
|
||||
if self._due_for_full_retrain(state):
|
||||
return self._full_decision(reason="full_retrain_due")
|
||||
return self._finetune_decision(state, reason="nightly_finetune")
|
||||
|
||||
def _due_for_full_retrain(self, state: TrainingRegimeState) -> bool:
|
||||
if state.last_full_completed_at is None:
|
||||
return True
|
||||
completed_at = datetime.fromisoformat(state.last_full_completed_at)
|
||||
due_after = completed_at + timedelta(
|
||||
days=self._full_interval_days(state.last_full_training_seconds)
|
||||
)
|
||||
return datetime.now(timezone.utc) >= due_after
|
||||
|
||||
def _full_decision(self, reason: str) -> dict[str, object]:
|
||||
return {
|
||||
"mode": "full",
|
||||
"reason": reason,
|
||||
"sample_limit": None,
|
||||
"epochs": self.config.full_epochs,
|
||||
"learning_rate": self.config.full_learning_rate,
|
||||
}
|
||||
|
||||
def _finetune_decision(
|
||||
self,
|
||||
state: TrainingRegimeState,
|
||||
reason: str,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"mode": "finetune",
|
||||
"reason": reason,
|
||||
"sample_limit": self._finetune_sample_limit(state),
|
||||
"epochs": self.config.finetune_epochs,
|
||||
"learning_rate": self.config.finetune_learning_rate,
|
||||
}
|
||||
|
||||
def _finetune_sample_limit(self, state: TrainingRegimeState) -> int | None:
|
||||
if not state.last_full_training_seconds or not state.last_full_sample_count:
|
||||
return None
|
||||
samples_per_second = state.last_full_sample_count / state.last_full_training_seconds
|
||||
sample_limit = round(samples_per_second * self.config.target_finetune_seconds)
|
||||
return max(self.config.finetune_min_samples, sample_limit)
|
||||
|
||||
def _full_interval_days(self, training_seconds: float | None) -> float:
|
||||
if training_seconds is None or training_seconds <= self.config.threshold_seconds:
|
||||
return self.config.nightly_interval_days
|
||||
if training_seconds >= self.config.terminal_threshold_seconds:
|
||||
return self.config.max_full_retrain_interval_days
|
||||
span = self.config.terminal_threshold_seconds - self.config.threshold_seconds
|
||||
progress = (training_seconds - self.config.threshold_seconds) / span
|
||||
return self.config.nightly_interval_days + progress * (
|
||||
self.config.max_full_retrain_interval_days - self.config.nightly_interval_days
|
||||
)
|
||||
|
||||
def _update_state(
|
||||
self,
|
||||
state: TrainingRegimeState,
|
||||
event: dict[str, object],
|
||||
) -> None:
|
||||
state.run_count += 1
|
||||
if event["mode"] == "full":
|
||||
state.last_full_training_seconds = float(event["training_seconds"])
|
||||
state.last_full_sample_count = int(event["sample_count"])
|
||||
state.last_full_completed_at = str(event["created_at"])
|
||||
if state.last_full_training_seconds >= self.config.terminal_threshold_seconds:
|
||||
state.full_retrain_locked = True
|
||||
else:
|
||||
state.last_finetune_completed_at = str(event["created_at"])
|
||||
state.history.append(event)
|
||||
state.history = state.history[-30:]
|
||||
|
||||
def _log_event(self, event: dict[str, object]) -> None:
|
||||
self.config.log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self.config.log_path.open("a") as handle:
|
||||
handle.write(json.dumps(event, sort_keys=True) + "\n")
|
||||
|
||||
def _now_iso(self) -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Adaptive full-retrain/fine-tune scheduler for the usage TCN."
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument(
|
||||
"--force-mode",
|
||||
choices=("full", "finetune"),
|
||||
default=None,
|
||||
help="Override the adaptive decision for this run.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
EnvLoader().load()
|
||||
args = parse_args()
|
||||
UsageSequenceTrainingScheduler(TrainingRegimeConfig.from_env()).run(
|
||||
force_mode=args.force_mode,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,158 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from os import environ
|
||||
from pathlib import Path
|
||||
|
||||
from astrape_ems.classes.env import EnvLoader
|
||||
from astrape_ems.classes.store import SigenStore
|
||||
from astrape_oracle.classes.weather.store import WeatherStore
|
||||
from gibil.classes.usage_sequence_dataset import (
|
||||
UsageSequenceDatasetBuilder,
|
||||
UsageSequenceDatasetConfig,
|
||||
)
|
||||
from gibil.classes.usage_training import train_model
|
||||
|
||||
|
||||
def main() -> None:
|
||||
EnvLoader().load()
|
||||
environ.setdefault("TORCH_CPP_LOG_LEVEL", "ERROR")
|
||||
args = parse_args()
|
||||
config = UsageSequenceDatasetConfig.from_env()
|
||||
builder = UsageSequenceDatasetBuilder(
|
||||
config=config,
|
||||
plant_history=SigenStore.from_env(),
|
||||
weather_history=WeatherStore.from_env(),
|
||||
)
|
||||
examples = builder.build(limit=args.limit)
|
||||
|
||||
print(f"usage_sequence_examples={len(examples)}")
|
||||
print(
|
||||
"minimum_history_hours="
|
||||
f"{builder.max_past_hours + config.future_hours}"
|
||||
)
|
||||
print(f"past_features={len(builder.past_feature_names)}")
|
||||
for scale in config.past_scales:
|
||||
print(
|
||||
f"past_scale={scale.name} "
|
||||
f"hours={scale.hours} "
|
||||
f"step_seconds={scale.step_seconds} "
|
||||
f"steps={builder.past_steps(scale)}"
|
||||
)
|
||||
print(f"future_steps={builder.future_steps}")
|
||||
print(f"future_features={len(builder.future_feature_names)}")
|
||||
|
||||
if examples:
|
||||
first = examples[0]
|
||||
last = examples[-1]
|
||||
print(f"first_issued_at={first.issued_at.isoformat()}")
|
||||
print(f"last_issued_at={last.issued_at.isoformat()}")
|
||||
for name, rows in first.past_by_scale.items():
|
||||
print(f"first_past_{name}_shape={len(rows)}x{len(rows[0])}")
|
||||
token_count = sum(
|
||||
len(tokens)
|
||||
for tokens in first.past_tokens_by_scale[name]
|
||||
)
|
||||
print(f"first_past_{name}_tokens={token_count}")
|
||||
print(
|
||||
"first_future_feature_shape="
|
||||
f"{len(first.future_features)}x{len(first.future_features[0])}"
|
||||
)
|
||||
print(
|
||||
"first_future_tokens="
|
||||
f"{sum(len(tokens) for tokens in first.future_tokens)}"
|
||||
)
|
||||
print(f"first_targets={len(first.targets)}")
|
||||
print(
|
||||
"first_target_preview="
|
||||
+ ",".join(f"{value:.0f}" for value in first.targets[:8])
|
||||
)
|
||||
|
||||
if args.dry_run:
|
||||
return
|
||||
|
||||
if not examples:
|
||||
raise SystemExit("No usage sequence examples available for training yet.")
|
||||
|
||||
result = train_model(
|
||||
examples=examples,
|
||||
builder=builder,
|
||||
epochs=args.epochs,
|
||||
batch_size=args.batch_size,
|
||||
learning_rate=args.learning_rate,
|
||||
artifact_path=args.artifact_path,
|
||||
seed=args.seed,
|
||||
base_artifact_path=args.base_artifact_path,
|
||||
mode=args.mode,
|
||||
)
|
||||
print(f"training_mode={result.mode}")
|
||||
print(f"training_examples={result.training_examples}")
|
||||
print(f"validation_examples={result.validation_examples}")
|
||||
if result.final_train_loss is not None:
|
||||
print(f"final_train_pinball_loss={result.final_train_loss:.4f}")
|
||||
if result.final_validation_loss is not None:
|
||||
print(f"final_validation_pinball_loss={result.final_validation_loss:.4f}")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Build training windows for the sequence usage oracle."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Only build examples and print dataset shape.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Optional maximum number of examples to build.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--epochs",
|
||||
type=int,
|
||||
default=20,
|
||||
help="Training epochs.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-size",
|
||||
type=int,
|
||||
default=32,
|
||||
help="Training batch size.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--learning-rate",
|
||||
type=float,
|
||||
default=0.001,
|
||||
help="Adam learning rate.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--artifact-path",
|
||||
type=Path,
|
||||
default=Path("models/usage_sequence_tcn_v1.pt"),
|
||||
help="Where to save the trained TCN artifact.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--base-artifact-path",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Existing artifact to continue training from.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
choices=("full", "finetune"),
|
||||
default="full",
|
||||
help="Training mode recorded in the saved artifact.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=7,
|
||||
help="Deterministic shuffle seed.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user