rpotter6298
ccd4e7a48c
The forecasting runtime gibil pilots: weather ingest (Open-Meteo forecasts and the Ecowitt station), the oracle and weather stores, the snapshot daemon, the forecast evaluator, and the TCN training scheduler. classes/datasets.py holds the shapes the stores return for display. They sit beside the stores deliberately: a store's return type belongs to the store, and keeping them with the renderers would mean this repo importing the dashboard, which is backwards. Extracted from a single repository where the forecasting engine, the database plumbing, the hardware client, the ingest daemons and the web UI all imported each other freely. Verified that every module imports with gibil resolved from its own installed checkout. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
255 lines
7.7 KiB
Python
255 lines
7.7 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import date, datetime, timezone
|
|
from typing import Any
|
|
from urllib.parse import urlencode
|
|
from urllib.request import urlopen
|
|
import json
|
|
|
|
from gibil.classes.models import (
|
|
WeatherForecastPoint,
|
|
WeatherForecastRun,
|
|
WeatherResolvedTruth,
|
|
)
|
|
|
|
|
|
class OpenMeteoClient:
|
|
"""Fetches external weather forecasts from Open-Meteo."""
|
|
|
|
base_url = "https://api.open-meteo.com/v1/forecast"
|
|
|
|
def build_url(
|
|
self,
|
|
latitude: float,
|
|
longitude: float,
|
|
forecast_hours: int = 48,
|
|
timezone_name: str = "UTC",
|
|
) -> str:
|
|
params = {
|
|
"latitude": latitude,
|
|
"longitude": longitude,
|
|
"hourly": ",".join(
|
|
[
|
|
"temperature_2m",
|
|
"shortwave_radiation",
|
|
"cloud_cover",
|
|
]
|
|
),
|
|
"forecast_hours": forecast_hours,
|
|
"timezone": timezone_name,
|
|
}
|
|
return f"{self.base_url}?{urlencode(params)}"
|
|
|
|
def fetch_forecast(
|
|
self,
|
|
latitude: float,
|
|
longitude: float,
|
|
forecast_hours: int = 48,
|
|
) -> WeatherForecastRun:
|
|
url = self.build_url(latitude, longitude, forecast_hours)
|
|
with urlopen(url, timeout=10) as response:
|
|
payload = json.loads(response.read().decode("utf-8"))
|
|
|
|
return OpenMeteoParser().parse_forecast(
|
|
payload=payload,
|
|
latitude=latitude,
|
|
longitude=longitude,
|
|
issued_at=datetime.now(timezone.utc),
|
|
)
|
|
|
|
|
|
class OpenMeteoArchiveClient:
|
|
"""Fetches historical weather data from Open-Meteo archive."""
|
|
|
|
base_url = "https://archive-api.open-meteo.com/v1/archive"
|
|
|
|
def build_url(
|
|
self,
|
|
latitude: float,
|
|
longitude: float,
|
|
start_date: date,
|
|
end_date: date,
|
|
timezone_name: str = "UTC",
|
|
) -> str:
|
|
params = {
|
|
"latitude": latitude,
|
|
"longitude": longitude,
|
|
"start_date": start_date.isoformat(),
|
|
"end_date": end_date.isoformat(),
|
|
"hourly": ",".join(
|
|
[
|
|
"temperature_2m",
|
|
"shortwave_radiation",
|
|
"cloud_cover",
|
|
]
|
|
),
|
|
"timezone": timezone_name,
|
|
}
|
|
return f"{self.base_url}?{urlencode(params)}"
|
|
|
|
def fetch_resolved_truth(
|
|
self,
|
|
latitude: float,
|
|
longitude: float,
|
|
start_date: date,
|
|
end_date: date,
|
|
) -> list[WeatherResolvedTruth]:
|
|
url = self.build_url(latitude, longitude, start_date, end_date)
|
|
with urlopen(url, timeout=20) as response:
|
|
payload = json.loads(response.read().decode("utf-8"))
|
|
|
|
return OpenMeteoArchiveParser().parse_resolved_truth(payload)
|
|
|
|
|
|
class OpenMeteoParser:
|
|
"""Converts Open-Meteo JSON into clean external forecast records."""
|
|
|
|
def parse_forecast(
|
|
self,
|
|
payload: dict[str, Any],
|
|
latitude: float,
|
|
longitude: float,
|
|
issued_at: datetime,
|
|
) -> WeatherForecastRun:
|
|
hourly = payload.get("hourly", {})
|
|
times = hourly.get("time", [])
|
|
temperatures = hourly.get("temperature_2m", [])
|
|
radiation = hourly.get("shortwave_radiation", [])
|
|
cloud_cover = hourly.get("cloud_cover", [])
|
|
|
|
points: list[WeatherForecastPoint] = []
|
|
for index, raw_time in enumerate(times):
|
|
target_at = self._parse_time(raw_time)
|
|
horizon_hours = max(
|
|
0, round((target_at - issued_at).total_seconds() / 3600)
|
|
)
|
|
|
|
points.append(
|
|
WeatherForecastPoint(
|
|
issued_at=issued_at,
|
|
target_at=target_at,
|
|
horizon_hours=horizon_hours,
|
|
temperature_c=self._at(temperatures, index),
|
|
shortwave_radiation_w_m2=self._at(radiation, index),
|
|
cloud_cover_pct=self._at(cloud_cover, index),
|
|
)
|
|
)
|
|
|
|
return WeatherForecastRun(
|
|
issued_at=issued_at,
|
|
source="open_meteo",
|
|
latitude=latitude,
|
|
longitude=longitude,
|
|
points=points,
|
|
)
|
|
|
|
def _parse_time(self, raw_time: str) -> datetime:
|
|
parsed = datetime.fromisoformat(raw_time)
|
|
if parsed.tzinfo is None:
|
|
return parsed.replace(tzinfo=timezone.utc)
|
|
return parsed.astimezone(timezone.utc)
|
|
|
|
def _at(self, values: list[Any], index: int) -> float | None:
|
|
if index >= len(values):
|
|
return None
|
|
|
|
value = values[index]
|
|
if value is None:
|
|
return None
|
|
|
|
return float(value)
|
|
|
|
|
|
class OpenMeteoArchiveParser:
|
|
"""Converts Open-Meteo archive JSON into resolved truth records."""
|
|
|
|
def parse_resolved_truth(self, payload: dict[str, Any]) -> list[WeatherResolvedTruth]:
|
|
hourly = payload.get("hourly", {})
|
|
times = hourly.get("time", [])
|
|
temperatures = hourly.get("temperature_2m", [])
|
|
radiation = hourly.get("shortwave_radiation", [])
|
|
cloud_cover = hourly.get("cloud_cover", [])
|
|
|
|
truth: list[WeatherResolvedTruth] = []
|
|
for index, raw_time in enumerate(times):
|
|
truth.append(
|
|
WeatherResolvedTruth(
|
|
resolved_at=self._parse_time(raw_time),
|
|
temperature_c=self._at(temperatures, index),
|
|
shortwave_radiation_w_m2=self._at(radiation, index),
|
|
cloud_cover_pct=self._at(cloud_cover, index),
|
|
source="open_meteo_archive",
|
|
)
|
|
)
|
|
|
|
return truth
|
|
|
|
def _parse_time(self, raw_time: str) -> datetime:
|
|
parsed = datetime.fromisoformat(raw_time)
|
|
if parsed.tzinfo is None:
|
|
return parsed.replace(tzinfo=timezone.utc)
|
|
return parsed.astimezone(timezone.utc)
|
|
|
|
def _at(self, values: list[Any], index: int) -> float | None:
|
|
if index >= len(values):
|
|
return None
|
|
|
|
value = values[index]
|
|
if value is None:
|
|
return None
|
|
|
|
return float(value)
|
|
|
|
|
|
class WeatherBuilder:
|
|
"""Builds a clean database-ready set of external weather forecast records."""
|
|
|
|
def build_forecast_run(
|
|
self,
|
|
source: str,
|
|
latitude: float,
|
|
longitude: float,
|
|
points: list[WeatherForecastPoint],
|
|
issued_at: datetime | None = None,
|
|
) -> WeatherForecastRun:
|
|
if issued_at is None:
|
|
issued_at = datetime.now(timezone.utc)
|
|
|
|
clean_points = [
|
|
WeatherForecastPoint(
|
|
issued_at=issued_at,
|
|
target_at=point.target_at,
|
|
horizon_hours=max(
|
|
0, round((point.target_at - issued_at).total_seconds() / 3600)
|
|
),
|
|
temperature_c=point.temperature_c,
|
|
shortwave_radiation_w_m2=point.shortwave_radiation_w_m2,
|
|
cloud_cover_pct=point.cloud_cover_pct,
|
|
source=source,
|
|
)
|
|
for point in sorted(points, key=lambda item: item.target_at)
|
|
]
|
|
|
|
return WeatherForecastRun(
|
|
issued_at=issued_at,
|
|
source=source,
|
|
latitude=latitude,
|
|
longitude=longitude,
|
|
points=clean_points,
|
|
)
|
|
|
|
def points_for_horizon(
|
|
self,
|
|
forecast_runs: list[WeatherForecastRun],
|
|
horizon_hours: int,
|
|
) -> list[WeatherForecastPoint]:
|
|
points: list[WeatherForecastPoint] = []
|
|
for run in forecast_runs:
|
|
points.extend(
|
|
point
|
|
for point in run.points
|
|
if point.horizon_hours == horizon_hours
|
|
)
|
|
|
|
return sorted(points, key=lambda point: point.target_at)
|