Add new daemons and debug scripts for Sigenergy and Oracle functionalities
- Implement `sigen_daemon.py` to poll Sigenergy plant metrics and store snapshots. - Create `web_daemon.py` for serving a web interface with various endpoints. - Add debug scripts: - `debug_duplicates.py` to find duplicate target times in forecast data. - `debug_energy_forecast.py` to print baseline energy forecast curves. - `debug_oracle_evaluations.py` to run the oracle evaluator. - `debug_sigen.py` to inspect stored Sigenergy plant snapshots. - `debug_weather.py` to trace resolved truth data. - `modbus_test.py` for exploring Sigenergy plants or inverters over Modbus TCP. - Introduce `oracle_evaluator.py` for evaluating stored oracle predictions against actuals. - Add TCN training scripts in `tcn` directory for training usage sequence models.
This commit is contained in:
rpotter6298
@@ -0,0 +1 @@
|
||||
"""Long-running Astrape service entrypoints."""
|
||||
@@ -0,0 +1,137 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from os import environ
|
||||
from sys import stderr
|
||||
from time import sleep
|
||||
|
||||
from gibil.classes.env_loader import EnvLoader
|
||||
from gibil.classes.weather.builder import (
|
||||
OpenMeteoArchiveClient,
|
||||
OpenMeteoClient,
|
||||
WeatherBuilder,
|
||||
)
|
||||
from gibil.classes.weather.store import WeatherStore
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DbDaemonConfig:
|
||||
latitude: float
|
||||
longitude: float
|
||||
forecast_hours: int
|
||||
truth_lookback_days: int
|
||||
truth_end_delay_days: int
|
||||
poll_seconds: int
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "DbDaemonConfig":
|
||||
return cls(
|
||||
latitude=float(_required_env("ASTRAPE_LATITUDE")),
|
||||
longitude=float(_required_env("ASTRAPE_LONGITUDE")),
|
||||
forecast_hours=int(environ.get("ASTRAPE_WEATHER_FORECAST_HOURS", "48")),
|
||||
truth_lookback_days=int(
|
||||
environ.get("ASTRAPE_WEATHER_TRUTH_LOOKBACK_DAYS", "14")
|
||||
),
|
||||
truth_end_delay_days=int(
|
||||
environ.get("ASTRAPE_WEATHER_TRUTH_END_DELAY_DAYS", "5")
|
||||
),
|
||||
poll_seconds=int(environ.get("ASTRAPE_WEATHER_POLL_SECONDS", "3600")),
|
||||
)
|
||||
|
||||
|
||||
class DbDaemon:
|
||||
"""Runs builder components that populate Astrape's database."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: DbDaemonConfig,
|
||||
weather_client: OpenMeteoClient,
|
||||
archive_client: OpenMeteoArchiveClient,
|
||||
weather_builder: WeatherBuilder,
|
||||
weather_store: WeatherStore,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.weather_client = weather_client
|
||||
self.archive_client = archive_client
|
||||
self.weather_builder = weather_builder
|
||||
self.weather_store = weather_store
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "DbDaemon":
|
||||
return cls(
|
||||
config=DbDaemonConfig.from_env(),
|
||||
weather_client=OpenMeteoClient(),
|
||||
archive_client=OpenMeteoArchiveClient(),
|
||||
weather_builder=WeatherBuilder(),
|
||||
weather_store=WeatherStore.from_env(),
|
||||
)
|
||||
|
||||
def initialize(self) -> None:
|
||||
self.weather_store.initialize()
|
||||
|
||||
def run_once(self) -> tuple[int, int]:
|
||||
raw_run = self.weather_client.fetch_forecast(
|
||||
latitude=self.config.latitude,
|
||||
longitude=self.config.longitude,
|
||||
forecast_hours=self.config.forecast_hours,
|
||||
)
|
||||
forecast_run = self.weather_builder.build_forecast_run(
|
||||
source=raw_run.source,
|
||||
latitude=raw_run.latitude,
|
||||
longitude=raw_run.longitude,
|
||||
points=raw_run.points,
|
||||
issued_at=raw_run.issued_at,
|
||||
)
|
||||
forecast_count = self.weather_store.save_forecast_run(forecast_run)
|
||||
zero_hour_truth_count = self.weather_store.save_zero_hour_forecast_as_truth(
|
||||
forecast_run
|
||||
)
|
||||
|
||||
today = datetime.now(timezone.utc).date()
|
||||
truth_end = today - timedelta(days=self.config.truth_end_delay_days)
|
||||
truth_start = truth_end - timedelta(days=self.config.truth_lookback_days)
|
||||
truth_points = self.archive_client.fetch_resolved_truth(
|
||||
latitude=self.config.latitude,
|
||||
longitude=self.config.longitude,
|
||||
start_date=truth_start,
|
||||
end_date=truth_end,
|
||||
)
|
||||
archive_truth_count = self.weather_store.save_resolved_truth(truth_points)
|
||||
|
||||
return forecast_count, zero_hour_truth_count + archive_truth_count
|
||||
|
||||
def run_forever(self) -> None:
|
||||
self.initialize()
|
||||
while True:
|
||||
forecast_count, truth_count = self.run_once()
|
||||
print(
|
||||
f"stored_weather_forecast_points={forecast_count} "
|
||||
f"stored_weather_resolved_truth={truth_count}",
|
||||
flush=True,
|
||||
)
|
||||
sleep(self.config.poll_seconds)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
EnvLoader().load()
|
||||
daemon = DbDaemon.from_env()
|
||||
daemon.run_forever()
|
||||
except Exception as error:
|
||||
print(f"db_daemon_startup_error={error}", file=stderr)
|
||||
raise SystemExit(1) from error
|
||||
|
||||
|
||||
def _required_env(name: str) -> str:
|
||||
value = environ.get(name)
|
||||
if not value:
|
||||
raise RuntimeError(
|
||||
f"{name} is required. Set ASTRAPE_DATABASE_URL, "
|
||||
"ASTRAPE_LATITUDE, and ASTRAPE_LONGITUDE before starting db_daemon."
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,115 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
from os import environ
|
||||
from sys import stderr
|
||||
from time import sleep
|
||||
|
||||
from gibil.classes.oracle.builder import EnergyOracleBuilder
|
||||
from gibil.classes.env_loader import EnvLoader
|
||||
from gibil.classes.oracle.store import OracleStore
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OracleDaemonConfig:
|
||||
poll_seconds: float
|
||||
evaluate_forecasts: bool
|
||||
evaluation_actual_window_minutes: float
|
||||
evaluation_lookback_hours: float
|
||||
evaluation_limit: int
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "OracleDaemonConfig":
|
||||
return cls(
|
||||
poll_seconds=float(environ.get("ASTRAPE_ORACLE_POLL_SECONDS", "300")),
|
||||
evaluate_forecasts=environ.get(
|
||||
"ASTRAPE_ORACLE_EVALUATE_FORECASTS", "1"
|
||||
).lower()
|
||||
not in {"0", "false", "no"},
|
||||
evaluation_actual_window_minutes=float(
|
||||
environ.get("ASTRAPE_ORACLE_EVALUATION_WINDOW_MINUTES", "5")
|
||||
),
|
||||
evaluation_lookback_hours=float(
|
||||
environ.get("ASTRAPE_ORACLE_EVALUATION_LOOKBACK_HOURS", "168")
|
||||
),
|
||||
evaluation_limit=int(environ.get("ASTRAPE_ORACLE_EVALUATION_LIMIT", "1000")),
|
||||
)
|
||||
|
||||
|
||||
class OracleDaemon:
|
||||
"""Periodically stores oracle projection curves for evaluation."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: OracleDaemonConfig,
|
||||
builder: EnergyOracleBuilder,
|
||||
store: OracleStore,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.builder = builder
|
||||
self.store = store
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "OracleDaemon":
|
||||
return cls(
|
||||
config=OracleDaemonConfig.from_env(),
|
||||
builder=EnergyOracleBuilder.from_env(),
|
||||
store=OracleStore.from_env(),
|
||||
)
|
||||
|
||||
def run_once(self) -> int:
|
||||
solar_run, load_run, net_run = self.builder.build()
|
||||
saved_count = self.store.save_runs(solar_run, load_run, net_run)
|
||||
if self.config.evaluate_forecasts:
|
||||
from datetime import timedelta
|
||||
|
||||
evaluated_count = self.store.evaluate_due_forecasts(
|
||||
actual_window=timedelta(
|
||||
minutes=self.config.evaluation_actual_window_minutes
|
||||
),
|
||||
lookback=timedelta(hours=self.config.evaluation_lookback_hours),
|
||||
limit=self.config.evaluation_limit,
|
||||
)
|
||||
return saved_count + evaluated_count
|
||||
return saved_count
|
||||
|
||||
def run_forever(self) -> None:
|
||||
self.store.initialize()
|
||||
while True:
|
||||
try:
|
||||
saved_count = self.run_once()
|
||||
print(f"stored_oracle_records={saved_count}", flush=True)
|
||||
except Exception as error:
|
||||
print(f"oracle_poll_error={error}", file=stderr, flush=True)
|
||||
sleep(self.config.poll_seconds)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
EnvLoader().load()
|
||||
args = parse_args()
|
||||
daemon = OracleDaemon.from_env()
|
||||
if args.once:
|
||||
print(f"stored_oracle_records={daemon.run_once()}", flush=True)
|
||||
return
|
||||
daemon.run_forever()
|
||||
except Exception as error:
|
||||
print(f"oracle_daemon_startup_error={error}", file=stderr)
|
||||
raise SystemExit(1) from error
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Store Astrape oracle projection curves."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--once",
|
||||
action="store_true",
|
||||
help="Save one set of oracle curves and exit.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,95 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
from os import environ
|
||||
from sys import stderr
|
||||
from time import sleep
|
||||
|
||||
from gibil.classes.env_loader import EnvLoader
|
||||
from gibil.classes.sigen.builder import SigenPlantClient
|
||||
from gibil.classes.sigen.store import SigenStore
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SigenDaemonConfig:
|
||||
poll_seconds: float
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "SigenDaemonConfig":
|
||||
return cls(
|
||||
poll_seconds=float(environ.get("SIGEN_POLL_SECONDS", "5")),
|
||||
)
|
||||
|
||||
|
||||
class SigenDaemon:
|
||||
"""Polls Sigenergy plant metrics and stores normalized snapshots."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: SigenDaemonConfig,
|
||||
plant_client: SigenPlantClient,
|
||||
sigen_store: SigenStore,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.plant_client = plant_client
|
||||
self.sigen_store = sigen_store
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "SigenDaemon":
|
||||
return cls(
|
||||
config=SigenDaemonConfig.from_env(),
|
||||
plant_client=SigenPlantClient.from_env(),
|
||||
sigen_store=SigenStore.from_env(),
|
||||
)
|
||||
|
||||
def initialize(self) -> None:
|
||||
self.sigen_store.initialize()
|
||||
|
||||
def run_once(self) -> int:
|
||||
snapshot = self.plant_client.fetch_snapshot()
|
||||
return self.sigen_store.save_snapshot(snapshot)
|
||||
|
||||
def run_forever(self) -> None:
|
||||
self.initialize()
|
||||
while True:
|
||||
try:
|
||||
saved_count = self.run_once()
|
||||
print(f"stored_sigen_plant_snapshots={saved_count}", flush=True)
|
||||
except Exception as error:
|
||||
print(f"sigen_poll_error={error}", file=stderr, flush=True)
|
||||
|
||||
sleep(self.config.poll_seconds)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
EnvLoader().load()
|
||||
daemon = SigenDaemon.from_env()
|
||||
args = parse_args()
|
||||
if args.once:
|
||||
daemon.initialize()
|
||||
saved_count = daemon.run_once()
|
||||
print(f"stored_sigen_plant_snapshots={saved_count}", flush=True)
|
||||
return
|
||||
|
||||
daemon.run_forever()
|
||||
except Exception as error:
|
||||
print(f"sigen_daemon_startup_error={error}", file=stderr)
|
||||
raise SystemExit(1) from error
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Poll Sigenergy plant metrics into Astrape's database."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--once",
|
||||
action="store_true",
|
||||
help="Initialize storage, save one snapshot, and exit.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,142 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from importlib import import_module, reload
|
||||
from os import environ
|
||||
from pathlib import Path
|
||||
import json
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from gibil.classes.env_loader import EnvLoader
|
||||
|
||||
EnvLoader().load()
|
||||
|
||||
HOST = environ.get("ASTRAPE_WEB_HOST", "0.0.0.0")
|
||||
PORT = int(environ.get("ASTRAPE_WEB_PORT", "8080"))
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
WATCHED_PATHS = [
|
||||
PROJECT_ROOT / "gibil" / "classes" / "webui.py",
|
||||
PROJECT_ROOT / "gibil" / "classes" / "weather" / "display.py",
|
||||
PROJECT_ROOT / "gibil" / "classes" / "oracle" / "display.py",
|
||||
PROJECT_ROOT / "gibil" / "classes" / "oracle" / "quality_display.py",
|
||||
PROJECT_ROOT / "gibil" / "classes" / "weather" / "store.py",
|
||||
PROJECT_ROOT / "gibil" / "classes" / "oracle" / "store.py",
|
||||
PROJECT_ROOT / "gibil" / "classes" / "oracle" / "builder.py",
|
||||
PROJECT_ROOT / "gibil" / "classes" / "oracle" / "config.py",
|
||||
PROJECT_ROOT / "gibil" / "classes" / "sigen" / "store.py",
|
||||
]
|
||||
|
||||
|
||||
class AstrapeWebHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None:
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path
|
||||
|
||||
if path in {"/", "/oracle"}:
|
||||
self._send_html(self._webui().render_page("oracle"))
|
||||
return
|
||||
|
||||
if path == "/weather":
|
||||
self._send_html(self._webui().render_page("weather"))
|
||||
return
|
||||
|
||||
if path == "/quality":
|
||||
self._send_html(self._webui().render_page("quality"))
|
||||
return
|
||||
|
||||
if path == "/api/weather":
|
||||
self._send_json_text(self._webui().weather_payload())
|
||||
return
|
||||
|
||||
if path == "/api/oracle":
|
||||
self._send_json_text(self._webui().oracle_payload())
|
||||
return
|
||||
|
||||
if path == "/api/oracle-quality":
|
||||
params = parse_qs(parsed.query)
|
||||
lookback_hours = self._float_param(params, "lookback_hours", 168)
|
||||
self._send_json_text(
|
||||
self._webui().oracle_quality_payload(
|
||||
lookback_hours=lookback_hours
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
if path == "/api/ui-version":
|
||||
self._send_json_text(json.dumps({"version": self._ui_version()}))
|
||||
return
|
||||
|
||||
self.send_error(404)
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
print(f"{self.address_string()} - {format % args}")
|
||||
|
||||
def _webui(self):
|
||||
weather_store_module = import_module("gibil.classes.weather.store")
|
||||
sigen_store_module = import_module("gibil.classes.sigen.store")
|
||||
oracle_store_module = import_module("gibil.classes.oracle.store")
|
||||
oracle_builder_module = import_module("gibil.classes.oracle.builder")
|
||||
oracle_display_module = import_module("gibil.classes.oracle.display")
|
||||
oracle_quality_display_module = import_module(
|
||||
"gibil.classes.oracle.quality_display"
|
||||
)
|
||||
weather_display_module = import_module("gibil.classes.weather.display")
|
||||
webui_module = import_module("gibil.classes.webui")
|
||||
reload(weather_store_module)
|
||||
reload(sigen_store_module)
|
||||
reload(oracle_store_module)
|
||||
reload(oracle_builder_module)
|
||||
reload(oracle_display_module)
|
||||
reload(oracle_quality_display_module)
|
||||
reload(weather_display_module)
|
||||
reload(webui_module)
|
||||
return webui_module.WebUI()
|
||||
|
||||
def _float_param(
|
||||
self,
|
||||
params: dict[str, list[str]],
|
||||
key: str,
|
||||
default: float,
|
||||
) -> float:
|
||||
values = params.get(key)
|
||||
if not values:
|
||||
return default
|
||||
try:
|
||||
return float(values[0])
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
def _ui_version(self) -> str:
|
||||
mtimes = [
|
||||
str(path.stat().st_mtime_ns)
|
||||
for path in WATCHED_PATHS
|
||||
if path.exists()
|
||||
]
|
||||
return ".".join(mtimes)
|
||||
|
||||
def _send_html(self, body: str) -> None:
|
||||
encoded = body.encode("utf-8")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/html; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(encoded)))
|
||||
self.end_headers()
|
||||
self.wfile.write(encoded)
|
||||
|
||||
def _send_json_text(self, body: str) -> None:
|
||||
encoded = body.encode("utf-8")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.send_header("Content-Length", str(len(encoded)))
|
||||
self.end_headers()
|
||||
self.wfile.write(encoded)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
server = ThreadingHTTPServer((HOST, PORT), AstrapeWebHandler)
|
||||
print(f"Astrape web UI listening on http://{HOST}:{PORT}")
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user