Add new daemons and debug scripts for Sigenergy and Oracle functionalities
- Implement `sigen_daemon.py` to poll Sigenergy plant metrics and store snapshots. - Create `web_daemon.py` for serving a web interface with various endpoints. - Add debug scripts: - `debug_duplicates.py` to find duplicate target times in forecast data. - `debug_energy_forecast.py` to print baseline energy forecast curves. - `debug_oracle_evaluations.py` to run the oracle evaluator. - `debug_sigen.py` to inspect stored Sigenergy plant snapshots. - `debug_weather.py` to trace resolved truth data. - `modbus_test.py` for exploring Sigenergy plants or inverters over Modbus TCP. - Introduce `oracle_evaluator.py` for evaluating stored oracle predictions against actuals. - Add TCN training scripts in `tcn` directory for training usage sequence models.
This commit is contained in:
rpotter6298
@@ -0,0 +1,254 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import urlopen
|
||||
import json
|
||||
|
||||
from gibil.classes.models import (
|
||||
WeatherForecastPoint,
|
||||
WeatherForecastRun,
|
||||
WeatherResolvedTruth,
|
||||
)
|
||||
|
||||
|
||||
class OpenMeteoClient:
|
||||
"""Fetches external weather forecasts from Open-Meteo."""
|
||||
|
||||
base_url = "https://api.open-meteo.com/v1/forecast"
|
||||
|
||||
def build_url(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
forecast_hours: int = 48,
|
||||
timezone_name: str = "UTC",
|
||||
) -> str:
|
||||
params = {
|
||||
"latitude": latitude,
|
||||
"longitude": longitude,
|
||||
"hourly": ",".join(
|
||||
[
|
||||
"temperature_2m",
|
||||
"shortwave_radiation",
|
||||
"cloud_cover",
|
||||
]
|
||||
),
|
||||
"forecast_hours": forecast_hours,
|
||||
"timezone": timezone_name,
|
||||
}
|
||||
return f"{self.base_url}?{urlencode(params)}"
|
||||
|
||||
def fetch_forecast(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
forecast_hours: int = 48,
|
||||
) -> WeatherForecastRun:
|
||||
url = self.build_url(latitude, longitude, forecast_hours)
|
||||
with urlopen(url, timeout=10) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
|
||||
return OpenMeteoParser().parse_forecast(
|
||||
payload=payload,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
issued_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
class OpenMeteoArchiveClient:
|
||||
"""Fetches historical weather data from Open-Meteo archive."""
|
||||
|
||||
base_url = "https://archive-api.open-meteo.com/v1/archive"
|
||||
|
||||
def build_url(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
start_date: date,
|
||||
end_date: date,
|
||||
timezone_name: str = "UTC",
|
||||
) -> str:
|
||||
params = {
|
||||
"latitude": latitude,
|
||||
"longitude": longitude,
|
||||
"start_date": start_date.isoformat(),
|
||||
"end_date": end_date.isoformat(),
|
||||
"hourly": ",".join(
|
||||
[
|
||||
"temperature_2m",
|
||||
"shortwave_radiation",
|
||||
"cloud_cover",
|
||||
]
|
||||
),
|
||||
"timezone": timezone_name,
|
||||
}
|
||||
return f"{self.base_url}?{urlencode(params)}"
|
||||
|
||||
def fetch_resolved_truth(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
start_date: date,
|
||||
end_date: date,
|
||||
) -> list[WeatherResolvedTruth]:
|
||||
url = self.build_url(latitude, longitude, start_date, end_date)
|
||||
with urlopen(url, timeout=20) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
|
||||
return OpenMeteoArchiveParser().parse_resolved_truth(payload)
|
||||
|
||||
|
||||
class OpenMeteoParser:
|
||||
"""Converts Open-Meteo JSON into clean external forecast records."""
|
||||
|
||||
def parse_forecast(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
issued_at: datetime,
|
||||
) -> WeatherForecastRun:
|
||||
hourly = payload.get("hourly", {})
|
||||
times = hourly.get("time", [])
|
||||
temperatures = hourly.get("temperature_2m", [])
|
||||
radiation = hourly.get("shortwave_radiation", [])
|
||||
cloud_cover = hourly.get("cloud_cover", [])
|
||||
|
||||
points: list[WeatherForecastPoint] = []
|
||||
for index, raw_time in enumerate(times):
|
||||
target_at = self._parse_time(raw_time)
|
||||
horizon_hours = max(
|
||||
0, round((target_at - issued_at).total_seconds() / 3600)
|
||||
)
|
||||
|
||||
points.append(
|
||||
WeatherForecastPoint(
|
||||
issued_at=issued_at,
|
||||
target_at=target_at,
|
||||
horizon_hours=horizon_hours,
|
||||
temperature_c=self._at(temperatures, index),
|
||||
shortwave_radiation_w_m2=self._at(radiation, index),
|
||||
cloud_cover_pct=self._at(cloud_cover, index),
|
||||
)
|
||||
)
|
||||
|
||||
return WeatherForecastRun(
|
||||
issued_at=issued_at,
|
||||
source="open_meteo",
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
points=points,
|
||||
)
|
||||
|
||||
def _parse_time(self, raw_time: str) -> datetime:
|
||||
parsed = datetime.fromisoformat(raw_time)
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
def _at(self, values: list[Any], index: int) -> float | None:
|
||||
if index >= len(values):
|
||||
return None
|
||||
|
||||
value = values[index]
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
return float(value)
|
||||
|
||||
|
||||
class OpenMeteoArchiveParser:
|
||||
"""Converts Open-Meteo archive JSON into resolved truth records."""
|
||||
|
||||
def parse_resolved_truth(self, payload: dict[str, Any]) -> list[WeatherResolvedTruth]:
|
||||
hourly = payload.get("hourly", {})
|
||||
times = hourly.get("time", [])
|
||||
temperatures = hourly.get("temperature_2m", [])
|
||||
radiation = hourly.get("shortwave_radiation", [])
|
||||
cloud_cover = hourly.get("cloud_cover", [])
|
||||
|
||||
truth: list[WeatherResolvedTruth] = []
|
||||
for index, raw_time in enumerate(times):
|
||||
truth.append(
|
||||
WeatherResolvedTruth(
|
||||
resolved_at=self._parse_time(raw_time),
|
||||
temperature_c=self._at(temperatures, index),
|
||||
shortwave_radiation_w_m2=self._at(radiation, index),
|
||||
cloud_cover_pct=self._at(cloud_cover, index),
|
||||
source="open_meteo_archive",
|
||||
)
|
||||
)
|
||||
|
||||
return truth
|
||||
|
||||
def _parse_time(self, raw_time: str) -> datetime:
|
||||
parsed = datetime.fromisoformat(raw_time)
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
def _at(self, values: list[Any], index: int) -> float | None:
|
||||
if index >= len(values):
|
||||
return None
|
||||
|
||||
value = values[index]
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
return float(value)
|
||||
|
||||
|
||||
class WeatherBuilder:
|
||||
"""Builds a clean database-ready set of external weather forecast records."""
|
||||
|
||||
def build_forecast_run(
|
||||
self,
|
||||
source: str,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
points: list[WeatherForecastPoint],
|
||||
issued_at: datetime | None = None,
|
||||
) -> WeatherForecastRun:
|
||||
if issued_at is None:
|
||||
issued_at = datetime.now(timezone.utc)
|
||||
|
||||
clean_points = [
|
||||
WeatherForecastPoint(
|
||||
issued_at=issued_at,
|
||||
target_at=point.target_at,
|
||||
horizon_hours=max(
|
||||
0, round((point.target_at - issued_at).total_seconds() / 3600)
|
||||
),
|
||||
temperature_c=point.temperature_c,
|
||||
shortwave_radiation_w_m2=point.shortwave_radiation_w_m2,
|
||||
cloud_cover_pct=point.cloud_cover_pct,
|
||||
source=source,
|
||||
)
|
||||
for point in sorted(points, key=lambda item: item.target_at)
|
||||
]
|
||||
|
||||
return WeatherForecastRun(
|
||||
issued_at=issued_at,
|
||||
source=source,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
points=clean_points,
|
||||
)
|
||||
|
||||
def points_for_horizon(
|
||||
self,
|
||||
forecast_runs: list[WeatherForecastRun],
|
||||
horizon_hours: int,
|
||||
) -> list[WeatherForecastPoint]:
|
||||
points: list[WeatherForecastPoint] = []
|
||||
for run in forecast_runs:
|
||||
points.extend(
|
||||
point
|
||||
for point in run.points
|
||||
if point.horizon_hours == horizon_hours
|
||||
)
|
||||
|
||||
return sorted(points, key=lambda point: point.target_at)
|
||||
Reference in New Issue
Block a user