Initial commit: astrape-oracle split out of the combined Astrape tree
The forecasting runtime gibil pilots: weather ingest (Open-Meteo forecasts and the Ecowitt station), the oracle and weather stores, the snapshot daemon, the forecast evaluator, and the TCN training scheduler. classes/datasets.py holds the shapes the stores return for display. They sit beside the stores deliberately: a store's return type belongs to the store, and keeping them with the renderers would mean this repo importing the dashboard, which is backwards. Extracted from a single repository where the forecasting engine, the database plumbing, the hardware client, the ingest daemons and the web UI all imported each other freely. Verified that every module imports with gibil resolved from its own installed checkout. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
rpotter6298
@@ -0,0 +1,151 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.request import urlopen
|
||||
import json
|
||||
import re
|
||||
|
||||
from gibil.classes.models import WeatherStationObservation
|
||||
|
||||
|
||||
class EcowittGw3000Client:
|
||||
"""Fetches local live data from an Ecowitt GW3000/GW3000A gateway."""
|
||||
|
||||
def __init__(self, host: str, timeout_seconds: float = 5) -> None:
|
||||
self.host = host.removeprefix("http://").removeprefix("https://").rstrip("/")
|
||||
self.timeout_seconds = timeout_seconds
|
||||
|
||||
def fetch_observation(self) -> WeatherStationObservation:
|
||||
received_at = datetime.now(timezone.utc)
|
||||
with urlopen(self._live_data_url(), timeout=self.timeout_seconds) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
|
||||
return EcowittGw3000Parser().parse(
|
||||
payload=payload,
|
||||
received_at=received_at,
|
||||
)
|
||||
|
||||
def _live_data_url(self) -> str:
|
||||
return f"http://{self.host}/get_livedata_info"
|
||||
|
||||
|
||||
class EcowittGw3000Parser:
|
||||
"""Normalizes GW3000 live-data JSON into a station observation."""
|
||||
|
||||
common_id_names = {
|
||||
"0x02": "temperature_c",
|
||||
"0x07": "relative_humidity_pct",
|
||||
"0x03": "dew_point_c",
|
||||
"0x0B": "wind_speed_m_s",
|
||||
"0x0C": "wind_gust_m_s",
|
||||
"0x19": "wind_max_m_s",
|
||||
"0x0A": "wind_direction_deg",
|
||||
"0x15": "solar_radiation_w_m2",
|
||||
"0x17": "uv_index",
|
||||
}
|
||||
rain_id_names = {
|
||||
"0x0D": "rain_event_mm",
|
||||
"0x0E": "rain_rate_mm_h",
|
||||
"0x7C": "rain_hourly_mm",
|
||||
"0x10": "rain_daily_mm",
|
||||
"0x11": "rain_weekly_mm",
|
||||
"0x12": "rain_monthly_mm",
|
||||
"0x13": "rain_yearly_mm",
|
||||
}
|
||||
|
||||
def parse(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
received_at: datetime | None = None,
|
||||
) -> WeatherStationObservation:
|
||||
if received_at is None:
|
||||
received_at = datetime.now(timezone.utc)
|
||||
|
||||
common = self._parse_id_list(payload.get("common_list", []))
|
||||
rain = self._parse_id_list(payload.get("rain", []))
|
||||
indoor = self._parse_indoor(payload.get("wh25", []))
|
||||
|
||||
return WeatherStationObservation(
|
||||
observed_at=received_at,
|
||||
received_at=received_at,
|
||||
source="ecowitt_gw3000",
|
||||
temperature_c=common.get("temperature_c"),
|
||||
relative_humidity_pct=common.get("relative_humidity_pct"),
|
||||
dew_point_c=common.get("dew_point_c"),
|
||||
wind_speed_m_s=common.get("wind_speed_m_s"),
|
||||
wind_gust_m_s=common.get("wind_gust_m_s"),
|
||||
wind_max_m_s=common.get("wind_max_m_s"),
|
||||
wind_direction_deg=common.get("wind_direction_deg"),
|
||||
solar_radiation_w_m2=common.get("solar_radiation_w_m2"),
|
||||
uv_index=common.get("uv_index"),
|
||||
rain_rate_mm_h=rain.get("rain_rate_mm_h"),
|
||||
rain_event_mm=rain.get("rain_event_mm"),
|
||||
rain_hourly_mm=rain.get("rain_hourly_mm"),
|
||||
rain_daily_mm=rain.get("rain_daily_mm"),
|
||||
rain_weekly_mm=rain.get("rain_weekly_mm"),
|
||||
rain_monthly_mm=rain.get("rain_monthly_mm"),
|
||||
rain_yearly_mm=rain.get("rain_yearly_mm"),
|
||||
indoor_temperature_c=indoor.get("indoor_temperature_c"),
|
||||
indoor_humidity_pct=indoor.get("indoor_humidity_pct"),
|
||||
absolute_pressure_hpa=indoor.get("absolute_pressure_hpa"),
|
||||
relative_pressure_hpa=indoor.get("relative_pressure_hpa"),
|
||||
metadata={
|
||||
"raw_payload": payload,
|
||||
"debug": payload.get("debug", []),
|
||||
},
|
||||
)
|
||||
|
||||
def _parse_id_list(self, rows: object) -> dict[str, float]:
|
||||
parsed: dict[str, float] = {}
|
||||
if not isinstance(rows, list):
|
||||
return parsed
|
||||
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
row_id = str(row.get("id", ""))
|
||||
key = self.common_id_names.get(row_id) or self.rain_id_names.get(row_id)
|
||||
if key is None:
|
||||
continue
|
||||
value = self._parse_number(row.get("val"))
|
||||
if value is None:
|
||||
continue
|
||||
if key in {"wind_speed_m_s", "wind_gust_m_s", "wind_max_m_s"}:
|
||||
value = self._knots_to_m_s(value)
|
||||
parsed[key] = value
|
||||
|
||||
return parsed
|
||||
|
||||
def _parse_indoor(self, rows: object) -> dict[str, float]:
|
||||
if not isinstance(rows, list) or not rows:
|
||||
return {}
|
||||
row = rows[0]
|
||||
if not isinstance(row, dict):
|
||||
return {}
|
||||
|
||||
values = {
|
||||
"indoor_temperature_c": self._parse_number(row.get("intemp")),
|
||||
"indoor_humidity_pct": self._parse_number(row.get("inhumi")),
|
||||
"absolute_pressure_hpa": self._parse_number(row.get("abs")),
|
||||
"relative_pressure_hpa": self._parse_number(row.get("rel")),
|
||||
}
|
||||
return {
|
||||
key: value
|
||||
for key, value in values.items()
|
||||
if value is not None
|
||||
}
|
||||
|
||||
def _parse_number(self, value: object) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
|
||||
match = re.search(r"-?\d+(?:\.\d+)?", str(value))
|
||||
if match is None:
|
||||
return None
|
||||
return float(match.group(0))
|
||||
|
||||
def _knots_to_m_s(self, value: float) -> float:
|
||||
return value * 0.514444
|
||||
@@ -0,0 +1,254 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import urlopen
|
||||
import json
|
||||
|
||||
from gibil.classes.models import (
|
||||
WeatherForecastPoint,
|
||||
WeatherForecastRun,
|
||||
WeatherResolvedTruth,
|
||||
)
|
||||
|
||||
|
||||
class OpenMeteoClient:
|
||||
"""Fetches external weather forecasts from Open-Meteo."""
|
||||
|
||||
base_url = "https://api.open-meteo.com/v1/forecast"
|
||||
|
||||
def build_url(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
forecast_hours: int = 48,
|
||||
timezone_name: str = "UTC",
|
||||
) -> str:
|
||||
params = {
|
||||
"latitude": latitude,
|
||||
"longitude": longitude,
|
||||
"hourly": ",".join(
|
||||
[
|
||||
"temperature_2m",
|
||||
"shortwave_radiation",
|
||||
"cloud_cover",
|
||||
]
|
||||
),
|
||||
"forecast_hours": forecast_hours,
|
||||
"timezone": timezone_name,
|
||||
}
|
||||
return f"{self.base_url}?{urlencode(params)}"
|
||||
|
||||
def fetch_forecast(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
forecast_hours: int = 48,
|
||||
) -> WeatherForecastRun:
|
||||
url = self.build_url(latitude, longitude, forecast_hours)
|
||||
with urlopen(url, timeout=10) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
|
||||
return OpenMeteoParser().parse_forecast(
|
||||
payload=payload,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
issued_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
class OpenMeteoArchiveClient:
|
||||
"""Fetches historical weather data from Open-Meteo archive."""
|
||||
|
||||
base_url = "https://archive-api.open-meteo.com/v1/archive"
|
||||
|
||||
def build_url(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
start_date: date,
|
||||
end_date: date,
|
||||
timezone_name: str = "UTC",
|
||||
) -> str:
|
||||
params = {
|
||||
"latitude": latitude,
|
||||
"longitude": longitude,
|
||||
"start_date": start_date.isoformat(),
|
||||
"end_date": end_date.isoformat(),
|
||||
"hourly": ",".join(
|
||||
[
|
||||
"temperature_2m",
|
||||
"shortwave_radiation",
|
||||
"cloud_cover",
|
||||
]
|
||||
),
|
||||
"timezone": timezone_name,
|
||||
}
|
||||
return f"{self.base_url}?{urlencode(params)}"
|
||||
|
||||
def fetch_resolved_truth(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
start_date: date,
|
||||
end_date: date,
|
||||
) -> list[WeatherResolvedTruth]:
|
||||
url = self.build_url(latitude, longitude, start_date, end_date)
|
||||
with urlopen(url, timeout=20) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
|
||||
return OpenMeteoArchiveParser().parse_resolved_truth(payload)
|
||||
|
||||
|
||||
class OpenMeteoParser:
|
||||
"""Converts Open-Meteo JSON into clean external forecast records."""
|
||||
|
||||
def parse_forecast(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
issued_at: datetime,
|
||||
) -> WeatherForecastRun:
|
||||
hourly = payload.get("hourly", {})
|
||||
times = hourly.get("time", [])
|
||||
temperatures = hourly.get("temperature_2m", [])
|
||||
radiation = hourly.get("shortwave_radiation", [])
|
||||
cloud_cover = hourly.get("cloud_cover", [])
|
||||
|
||||
points: list[WeatherForecastPoint] = []
|
||||
for index, raw_time in enumerate(times):
|
||||
target_at = self._parse_time(raw_time)
|
||||
horizon_hours = max(
|
||||
0, round((target_at - issued_at).total_seconds() / 3600)
|
||||
)
|
||||
|
||||
points.append(
|
||||
WeatherForecastPoint(
|
||||
issued_at=issued_at,
|
||||
target_at=target_at,
|
||||
horizon_hours=horizon_hours,
|
||||
temperature_c=self._at(temperatures, index),
|
||||
shortwave_radiation_w_m2=self._at(radiation, index),
|
||||
cloud_cover_pct=self._at(cloud_cover, index),
|
||||
)
|
||||
)
|
||||
|
||||
return WeatherForecastRun(
|
||||
issued_at=issued_at,
|
||||
source="open_meteo",
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
points=points,
|
||||
)
|
||||
|
||||
def _parse_time(self, raw_time: str) -> datetime:
|
||||
parsed = datetime.fromisoformat(raw_time)
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
def _at(self, values: list[Any], index: int) -> float | None:
|
||||
if index >= len(values):
|
||||
return None
|
||||
|
||||
value = values[index]
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
return float(value)
|
||||
|
||||
|
||||
class OpenMeteoArchiveParser:
|
||||
"""Converts Open-Meteo archive JSON into resolved truth records."""
|
||||
|
||||
def parse_resolved_truth(self, payload: dict[str, Any]) -> list[WeatherResolvedTruth]:
|
||||
hourly = payload.get("hourly", {})
|
||||
times = hourly.get("time", [])
|
||||
temperatures = hourly.get("temperature_2m", [])
|
||||
radiation = hourly.get("shortwave_radiation", [])
|
||||
cloud_cover = hourly.get("cloud_cover", [])
|
||||
|
||||
truth: list[WeatherResolvedTruth] = []
|
||||
for index, raw_time in enumerate(times):
|
||||
truth.append(
|
||||
WeatherResolvedTruth(
|
||||
resolved_at=self._parse_time(raw_time),
|
||||
temperature_c=self._at(temperatures, index),
|
||||
shortwave_radiation_w_m2=self._at(radiation, index),
|
||||
cloud_cover_pct=self._at(cloud_cover, index),
|
||||
source="open_meteo_archive",
|
||||
)
|
||||
)
|
||||
|
||||
return truth
|
||||
|
||||
def _parse_time(self, raw_time: str) -> datetime:
|
||||
parsed = datetime.fromisoformat(raw_time)
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
def _at(self, values: list[Any], index: int) -> float | None:
|
||||
if index >= len(values):
|
||||
return None
|
||||
|
||||
value = values[index]
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
return float(value)
|
||||
|
||||
|
||||
class WeatherBuilder:
|
||||
"""Builds a clean database-ready set of external weather forecast records."""
|
||||
|
||||
def build_forecast_run(
|
||||
self,
|
||||
source: str,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
points: list[WeatherForecastPoint],
|
||||
issued_at: datetime | None = None,
|
||||
) -> WeatherForecastRun:
|
||||
if issued_at is None:
|
||||
issued_at = datetime.now(timezone.utc)
|
||||
|
||||
clean_points = [
|
||||
WeatherForecastPoint(
|
||||
issued_at=issued_at,
|
||||
target_at=point.target_at,
|
||||
horizon_hours=max(
|
||||
0, round((point.target_at - issued_at).total_seconds() / 3600)
|
||||
),
|
||||
temperature_c=point.temperature_c,
|
||||
shortwave_radiation_w_m2=point.shortwave_radiation_w_m2,
|
||||
cloud_cover_pct=point.cloud_cover_pct,
|
||||
source=source,
|
||||
)
|
||||
for point in sorted(points, key=lambda item: item.target_at)
|
||||
]
|
||||
|
||||
return WeatherForecastRun(
|
||||
issued_at=issued_at,
|
||||
source=source,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
points=clean_points,
|
||||
)
|
||||
|
||||
def points_for_horizon(
|
||||
self,
|
||||
forecast_runs: list[WeatherForecastRun],
|
||||
horizon_hours: int,
|
||||
) -> list[WeatherForecastPoint]:
|
||||
points: list[WeatherForecastPoint] = []
|
||||
for run in forecast_runs:
|
||||
points.extend(
|
||||
point
|
||||
for point in run.points
|
||||
if point.horizon_hours == horizon_hours
|
||||
)
|
||||
|
||||
return sorted(points, key=lambda point: point.target_at)
|
||||
@@ -0,0 +1,640 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from os import environ
|
||||
from typing import Iterator
|
||||
|
||||
from gibil.classes.models import (
|
||||
WeatherForecastPoint,
|
||||
WeatherForecastRun,
|
||||
WeatherResolvedTruth,
|
||||
WeatherStationObservation,
|
||||
)
|
||||
from astrape_oracle.classes.datasets import (
|
||||
WeatherDisplayDataset,
|
||||
WeatherStationDisplayDataset,
|
||||
)
|
||||
|
||||
|
||||
class WeatherStoreConfigurationError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WeatherStoreConfig:
|
||||
database_url: str
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "WeatherStoreConfig":
|
||||
database_url = environ.get("ASTRAPE_DATABASE_URL")
|
||||
if not database_url:
|
||||
raise WeatherStoreConfigurationError(
|
||||
"ASTRAPE_DATABASE_URL is required for weather storage"
|
||||
)
|
||||
|
||||
return cls(database_url=database_url)
|
||||
|
||||
|
||||
class WeatherStore:
|
||||
"""Persists external weather forecasts and resolved truth in TimescaleDB."""
|
||||
|
||||
def __init__(self, config: WeatherStoreConfig) -> None:
|
||||
self.config = config
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "WeatherStore":
|
||||
return cls(WeatherStoreConfig.from_env())
|
||||
|
||||
def initialize(self) -> None:
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("CREATE EXTENSION IF NOT EXISTS timescaledb")
|
||||
cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS weather_forecast_points (
|
||||
issued_at TIMESTAMPTZ NOT NULL,
|
||||
target_at TIMESTAMPTZ NOT NULL,
|
||||
horizon_hours INTEGER NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
latitude DOUBLE PRECISION NOT NULL,
|
||||
longitude DOUBLE PRECISION NOT NULL,
|
||||
temperature_c DOUBLE PRECISION,
|
||||
shortwave_radiation_w_m2 DOUBLE PRECISION,
|
||||
cloud_cover_pct DOUBLE PRECISION,
|
||||
inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (issued_at, target_at, source)
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT create_hypertable(
|
||||
'weather_forecast_points',
|
||||
'target_at',
|
||||
if_not_exists => TRUE
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS weather_resolved_truth (
|
||||
resolved_at TIMESTAMPTZ NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
temperature_c DOUBLE PRECISION,
|
||||
shortwave_radiation_w_m2 DOUBLE PRECISION,
|
||||
cloud_cover_pct DOUBLE PRECISION,
|
||||
inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (resolved_at, source)
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
ALTER TABLE weather_resolved_truth
|
||||
ADD COLUMN IF NOT EXISTS cloud_cover_pct DOUBLE PRECISION
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT create_hypertable(
|
||||
'weather_resolved_truth',
|
||||
'resolved_at',
|
||||
if_not_exists => TRUE
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS weather_station_observations (
|
||||
observed_at TIMESTAMPTZ NOT NULL,
|
||||
received_at TIMESTAMPTZ NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
temperature_c DOUBLE PRECISION,
|
||||
relative_humidity_pct DOUBLE PRECISION,
|
||||
dew_point_c DOUBLE PRECISION,
|
||||
pressure_hpa DOUBLE PRECISION,
|
||||
wind_speed_m_s DOUBLE PRECISION,
|
||||
wind_gust_m_s DOUBLE PRECISION,
|
||||
wind_max_m_s DOUBLE PRECISION,
|
||||
wind_direction_deg DOUBLE PRECISION,
|
||||
solar_radiation_w_m2 DOUBLE PRECISION,
|
||||
uv_index DOUBLE PRECISION,
|
||||
rain_rate_mm_h DOUBLE PRECISION,
|
||||
rain_event_mm DOUBLE PRECISION,
|
||||
rain_hourly_mm DOUBLE PRECISION,
|
||||
rain_daily_mm DOUBLE PRECISION,
|
||||
rain_weekly_mm DOUBLE PRECISION,
|
||||
rain_monthly_mm DOUBLE PRECISION,
|
||||
rain_yearly_mm DOUBLE PRECISION,
|
||||
indoor_temperature_c DOUBLE PRECISION,
|
||||
indoor_humidity_pct DOUBLE PRECISION,
|
||||
absolute_pressure_hpa DOUBLE PRECISION,
|
||||
relative_pressure_hpa DOUBLE PRECISION,
|
||||
metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (observed_at, source)
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT create_hypertable(
|
||||
'weather_station_observations',
|
||||
'observed_at',
|
||||
if_not_exists => TRUE
|
||||
)
|
||||
"""
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
def save_forecast_run(self, forecast_run: WeatherForecastRun) -> int:
|
||||
rows = [
|
||||
(
|
||||
point.issued_at,
|
||||
point.target_at,
|
||||
point.horizon_hours,
|
||||
forecast_run.source,
|
||||
forecast_run.latitude,
|
||||
forecast_run.longitude,
|
||||
point.temperature_c,
|
||||
point.shortwave_radiation_w_m2,
|
||||
point.cloud_cover_pct,
|
||||
)
|
||||
for point in forecast_run.points
|
||||
]
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.executemany(
|
||||
"""
|
||||
INSERT INTO weather_forecast_points (
|
||||
issued_at,
|
||||
target_at,
|
||||
horizon_hours,
|
||||
source,
|
||||
latitude,
|
||||
longitude,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (issued_at, target_at, source)
|
||||
DO UPDATE SET
|
||||
horizon_hours = EXCLUDED.horizon_hours,
|
||||
latitude = EXCLUDED.latitude,
|
||||
longitude = EXCLUDED.longitude,
|
||||
temperature_c = EXCLUDED.temperature_c,
|
||||
shortwave_radiation_w_m2 = EXCLUDED.shortwave_radiation_w_m2,
|
||||
cloud_cover_pct = EXCLUDED.cloud_cover_pct,
|
||||
inserted_at = now()
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
return len(rows)
|
||||
|
||||
def save_resolved_truth(self, truth_points: list[WeatherResolvedTruth]) -> int:
|
||||
rows = [
|
||||
(
|
||||
point.resolved_at,
|
||||
point.source,
|
||||
point.temperature_c,
|
||||
point.shortwave_radiation_w_m2,
|
||||
point.cloud_cover_pct,
|
||||
)
|
||||
for point in truth_points
|
||||
]
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.executemany(
|
||||
"""
|
||||
INSERT INTO weather_resolved_truth (
|
||||
resolved_at,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s)
|
||||
ON CONFLICT (resolved_at, source)
|
||||
DO UPDATE SET
|
||||
temperature_c = EXCLUDED.temperature_c,
|
||||
shortwave_radiation_w_m2 = EXCLUDED.shortwave_radiation_w_m2,
|
||||
cloud_cover_pct = EXCLUDED.cloud_cover_pct,
|
||||
inserted_at = now()
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
return len(rows)
|
||||
|
||||
def save_zero_hour_forecast_as_truth(
|
||||
self, forecast_run: WeatherForecastRun
|
||||
) -> int:
|
||||
truth_points = [
|
||||
WeatherResolvedTruth(
|
||||
resolved_at=point.issued_at,
|
||||
source="open_meteo_zero_hour",
|
||||
temperature_c=point.temperature_c,
|
||||
shortwave_radiation_w_m2=point.shortwave_radiation_w_m2,
|
||||
cloud_cover_pct=point.cloud_cover_pct,
|
||||
)
|
||||
for point in forecast_run.points
|
||||
if point.horizon_hours == 0
|
||||
]
|
||||
return self.save_resolved_truth(truth_points)
|
||||
|
||||
def save_station_observation(
|
||||
self,
|
||||
observation: WeatherStationObservation,
|
||||
) -> int:
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
try:
|
||||
from psycopg.types.json import Jsonb
|
||||
except ImportError as error:
|
||||
raise WeatherStoreConfigurationError(
|
||||
"Install dependencies with `python3 -m pip install -r requirements.txt`"
|
||||
) from error
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO weather_station_observations (
|
||||
observed_at,
|
||||
received_at,
|
||||
source,
|
||||
temperature_c,
|
||||
relative_humidity_pct,
|
||||
dew_point_c,
|
||||
pressure_hpa,
|
||||
wind_speed_m_s,
|
||||
wind_gust_m_s,
|
||||
wind_max_m_s,
|
||||
wind_direction_deg,
|
||||
solar_radiation_w_m2,
|
||||
uv_index,
|
||||
rain_rate_mm_h,
|
||||
rain_event_mm,
|
||||
rain_hourly_mm,
|
||||
rain_daily_mm,
|
||||
rain_weekly_mm,
|
||||
rain_monthly_mm,
|
||||
rain_yearly_mm,
|
||||
indoor_temperature_c,
|
||||
indoor_humidity_pct,
|
||||
absolute_pressure_hpa,
|
||||
relative_pressure_hpa,
|
||||
metadata
|
||||
)
|
||||
VALUES (
|
||||
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
|
||||
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
|
||||
%s, %s, %s, %s, %s
|
||||
)
|
||||
ON CONFLICT (observed_at, source)
|
||||
DO UPDATE SET
|
||||
received_at = EXCLUDED.received_at,
|
||||
temperature_c = EXCLUDED.temperature_c,
|
||||
relative_humidity_pct = EXCLUDED.relative_humidity_pct,
|
||||
dew_point_c = EXCLUDED.dew_point_c,
|
||||
pressure_hpa = EXCLUDED.pressure_hpa,
|
||||
wind_speed_m_s = EXCLUDED.wind_speed_m_s,
|
||||
wind_gust_m_s = EXCLUDED.wind_gust_m_s,
|
||||
wind_max_m_s = EXCLUDED.wind_max_m_s,
|
||||
wind_direction_deg = EXCLUDED.wind_direction_deg,
|
||||
solar_radiation_w_m2 = EXCLUDED.solar_radiation_w_m2,
|
||||
uv_index = EXCLUDED.uv_index,
|
||||
rain_rate_mm_h = EXCLUDED.rain_rate_mm_h,
|
||||
rain_event_mm = EXCLUDED.rain_event_mm,
|
||||
rain_hourly_mm = EXCLUDED.rain_hourly_mm,
|
||||
rain_daily_mm = EXCLUDED.rain_daily_mm,
|
||||
rain_weekly_mm = EXCLUDED.rain_weekly_mm,
|
||||
rain_monthly_mm = EXCLUDED.rain_monthly_mm,
|
||||
rain_yearly_mm = EXCLUDED.rain_yearly_mm,
|
||||
indoor_temperature_c = EXCLUDED.indoor_temperature_c,
|
||||
indoor_humidity_pct = EXCLUDED.indoor_humidity_pct,
|
||||
absolute_pressure_hpa = EXCLUDED.absolute_pressure_hpa,
|
||||
relative_pressure_hpa = EXCLUDED.relative_pressure_hpa,
|
||||
metadata = EXCLUDED.metadata,
|
||||
inserted_at = now()
|
||||
""",
|
||||
(
|
||||
observation.observed_at,
|
||||
observation.received_at,
|
||||
observation.source,
|
||||
observation.temperature_c,
|
||||
observation.relative_humidity_pct,
|
||||
observation.dew_point_c,
|
||||
observation.pressure_hpa,
|
||||
observation.wind_speed_m_s,
|
||||
observation.wind_gust_m_s,
|
||||
observation.wind_max_m_s,
|
||||
observation.wind_direction_deg,
|
||||
observation.solar_radiation_w_m2,
|
||||
observation.uv_index,
|
||||
observation.rain_rate_mm_h,
|
||||
observation.rain_event_mm,
|
||||
observation.rain_hourly_mm,
|
||||
observation.rain_daily_mm,
|
||||
observation.rain_weekly_mm,
|
||||
observation.rain_monthly_mm,
|
||||
observation.rain_yearly_mm,
|
||||
observation.indoor_temperature_c,
|
||||
observation.indoor_humidity_pct,
|
||||
observation.absolute_pressure_hpa,
|
||||
observation.relative_pressure_hpa,
|
||||
Jsonb(observation.metadata),
|
||||
),
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
return 1
|
||||
|
||||
def load_latest_forecast_points(
|
||||
self,
|
||||
start_at: datetime,
|
||||
end_at: datetime,
|
||||
) -> list[WeatherForecastPoint]:
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
issued_at,
|
||||
target_at,
|
||||
horizon_hours,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
FROM (
|
||||
SELECT
|
||||
issued_at,
|
||||
target_at,
|
||||
horizon_hours,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY target_at
|
||||
ORDER BY issued_at DESC
|
||||
) as rn
|
||||
FROM weather_forecast_points
|
||||
WHERE target_at >= %s AND target_at <= %s
|
||||
) as ranked
|
||||
WHERE rn = 1
|
||||
ORDER BY target_at
|
||||
LIMIT 5000
|
||||
""",
|
||||
(start_at, end_at),
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
return [
|
||||
WeatherForecastPoint(
|
||||
issued_at=row[0],
|
||||
target_at=row[1],
|
||||
horizon_hours=row[2],
|
||||
source=row[3],
|
||||
temperature_c=row[4],
|
||||
shortwave_radiation_w_m2=row[5],
|
||||
cloud_cover_pct=row[6],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def load_forecast_history(
|
||||
self,
|
||||
start_at: datetime,
|
||||
end_at: datetime,
|
||||
) -> dict[datetime, list[dict[str, object]]]:
|
||||
"""Every forecast issued for each target in the window, oldest first.
|
||||
|
||||
Unlike `load_latest_forecast_points`, this keeps all vintages of a
|
||||
forecast, so a training example can be shown the forecast that was
|
||||
actually available at the time it was issued rather than the best one
|
||||
known in hindsight.
|
||||
"""
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
issued_at,
|
||||
target_at,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
FROM weather_forecast_points
|
||||
WHERE target_at >= %s
|
||||
AND target_at <= %s
|
||||
ORDER BY target_at, issued_at
|
||||
""",
|
||||
(start_at, end_at),
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
by_target: dict[datetime, list[dict[str, object]]] = {}
|
||||
for row in rows:
|
||||
by_target.setdefault(row[1], []).append(
|
||||
{
|
||||
"issued_at": row[0],
|
||||
"target_at": row[1],
|
||||
"temperature_c": row[2],
|
||||
"shortwave_radiation_w_m2": row[3],
|
||||
"cloud_cover_pct": row[4],
|
||||
}
|
||||
)
|
||||
return by_target
|
||||
|
||||
def load_display_dataset(
|
||||
self,
|
||||
start_at: datetime | None = None,
|
||||
end_at: datetime | None = None,
|
||||
) -> WeatherDisplayDataset:
|
||||
now = datetime.now(timezone.utc)
|
||||
if start_at is None:
|
||||
start_at = now - timedelta(hours=24)
|
||||
if end_at is None:
|
||||
end_at = now + timedelta(hours=48)
|
||||
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
issued_at,
|
||||
target_at,
|
||||
horizon_hours,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
FROM (
|
||||
SELECT
|
||||
issued_at,
|
||||
target_at,
|
||||
horizon_hours,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct,
|
||||
ROW_NUMBER() OVER (PARTITION BY target_at, horizon_hours ORDER BY issued_at DESC) as rn
|
||||
FROM weather_forecast_points
|
||||
WHERE target_at >= %s AND target_at <= %s
|
||||
) as ranked
|
||||
WHERE rn = 1
|
||||
ORDER BY target_at, horizon_hours
|
||||
LIMIT 5000
|
||||
""",
|
||||
(start_at, end_at),
|
||||
)
|
||||
forecast_rows = cursor.fetchall()
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
resolved_at,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
FROM weather_resolved_truth
|
||||
WHERE resolved_at >= %s AND resolved_at <= %s
|
||||
ORDER BY resolved_at
|
||||
LIMIT 5000
|
||||
""",
|
||||
(start_at, now),
|
||||
)
|
||||
truth_rows = cursor.fetchall()
|
||||
|
||||
return WeatherDisplayDataset(
|
||||
forecast_points=[
|
||||
WeatherForecastPoint(
|
||||
issued_at=row[0],
|
||||
target_at=row[1],
|
||||
horizon_hours=row[2],
|
||||
source=row[3],
|
||||
temperature_c=row[4],
|
||||
shortwave_radiation_w_m2=row[5],
|
||||
cloud_cover_pct=row[6],
|
||||
)
|
||||
for row in forecast_rows
|
||||
],
|
||||
resolved_truth=[
|
||||
WeatherResolvedTruth(
|
||||
resolved_at=row[0],
|
||||
source=row[1],
|
||||
temperature_c=row[2],
|
||||
shortwave_radiation_w_m2=row[3],
|
||||
cloud_cover_pct=row[4],
|
||||
)
|
||||
for row in truth_rows
|
||||
],
|
||||
)
|
||||
|
||||
def load_station_display_dataset(
|
||||
self,
|
||||
start_at: datetime | None = None,
|
||||
end_at: datetime | None = None,
|
||||
) -> WeatherStationDisplayDataset:
|
||||
now = datetime.now(timezone.utc)
|
||||
if start_at is None:
|
||||
start_at = now - timedelta(days=7)
|
||||
if end_at is None:
|
||||
end_at = now
|
||||
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
observed_at,
|
||||
received_at,
|
||||
source,
|
||||
temperature_c,
|
||||
relative_humidity_pct,
|
||||
dew_point_c,
|
||||
pressure_hpa,
|
||||
wind_speed_m_s,
|
||||
wind_gust_m_s,
|
||||
wind_max_m_s,
|
||||
wind_direction_deg,
|
||||
solar_radiation_w_m2,
|
||||
uv_index,
|
||||
rain_rate_mm_h,
|
||||
rain_event_mm,
|
||||
rain_hourly_mm,
|
||||
rain_daily_mm,
|
||||
rain_weekly_mm,
|
||||
rain_monthly_mm,
|
||||
rain_yearly_mm,
|
||||
indoor_temperature_c,
|
||||
indoor_humidity_pct,
|
||||
absolute_pressure_hpa,
|
||||
relative_pressure_hpa,
|
||||
metadata
|
||||
FROM weather_station_observations
|
||||
WHERE observed_at >= %s AND observed_at <= %s
|
||||
ORDER BY observed_at
|
||||
LIMIT 100000
|
||||
""",
|
||||
(start_at, end_at),
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
return WeatherStationDisplayDataset(
|
||||
observations=[
|
||||
WeatherStationObservation(
|
||||
observed_at=row[0],
|
||||
received_at=row[1],
|
||||
source=row[2],
|
||||
temperature_c=row[3],
|
||||
relative_humidity_pct=row[4],
|
||||
dew_point_c=row[5],
|
||||
pressure_hpa=row[6],
|
||||
wind_speed_m_s=row[7],
|
||||
wind_gust_m_s=row[8],
|
||||
wind_max_m_s=row[9],
|
||||
wind_direction_deg=row[10],
|
||||
solar_radiation_w_m2=row[11],
|
||||
uv_index=row[12],
|
||||
rain_rate_mm_h=row[13],
|
||||
rain_event_mm=row[14],
|
||||
rain_hourly_mm=row[15],
|
||||
rain_daily_mm=row[16],
|
||||
rain_weekly_mm=row[17],
|
||||
rain_monthly_mm=row[18],
|
||||
rain_yearly_mm=row[19],
|
||||
indoor_temperature_c=row[20],
|
||||
indoor_humidity_pct=row[21],
|
||||
absolute_pressure_hpa=row[22],
|
||||
relative_pressure_hpa=row[23],
|
||||
metadata=row[24] or {},
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def _connection(self) -> Iterator[object]:
|
||||
try:
|
||||
import psycopg
|
||||
except ImportError as error:
|
||||
raise WeatherStoreConfigurationError(
|
||||
"Install dependencies with `python3 -m pip install -r requirements.txt`"
|
||||
) from error
|
||||
|
||||
with psycopg.connect(self.config.database_url) as connection:
|
||||
yield connection
|
||||
Reference in New Issue
Block a user