Flatten to a single package directory
The checkout is now astrape/astrape-oracle, not astrape/astrape-oracle/<package>. The package contents move to the repository root and pyproject maps the package name to '.' via package-dir, with the subpackages listed explicitly. The systemd units move out to /srv/astrape/deploy/systemd, outside every repository, because they describe this deployment rather than any one component. Verified both install modes: editable, which is how the deployment runs, and a plain wheel install, which is what the container build does. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
rpotter6298
@@ -0,0 +1,151 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.request import urlopen
|
||||
import json
|
||||
import re
|
||||
|
||||
from gibil.classes.models import WeatherStationObservation
|
||||
|
||||
|
||||
class EcowittGw3000Client:
|
||||
"""Fetches local live data from an Ecowitt GW3000/GW3000A gateway."""
|
||||
|
||||
def __init__(self, host: str, timeout_seconds: float = 5) -> None:
|
||||
self.host = host.removeprefix("http://").removeprefix("https://").rstrip("/")
|
||||
self.timeout_seconds = timeout_seconds
|
||||
|
||||
def fetch_observation(self) -> WeatherStationObservation:
|
||||
received_at = datetime.now(timezone.utc)
|
||||
with urlopen(self._live_data_url(), timeout=self.timeout_seconds) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
|
||||
return EcowittGw3000Parser().parse(
|
||||
payload=payload,
|
||||
received_at=received_at,
|
||||
)
|
||||
|
||||
def _live_data_url(self) -> str:
|
||||
return f"http://{self.host}/get_livedata_info"
|
||||
|
||||
|
||||
class EcowittGw3000Parser:
|
||||
"""Normalizes GW3000 live-data JSON into a station observation."""
|
||||
|
||||
common_id_names = {
|
||||
"0x02": "temperature_c",
|
||||
"0x07": "relative_humidity_pct",
|
||||
"0x03": "dew_point_c",
|
||||
"0x0B": "wind_speed_m_s",
|
||||
"0x0C": "wind_gust_m_s",
|
||||
"0x19": "wind_max_m_s",
|
||||
"0x0A": "wind_direction_deg",
|
||||
"0x15": "solar_radiation_w_m2",
|
||||
"0x17": "uv_index",
|
||||
}
|
||||
rain_id_names = {
|
||||
"0x0D": "rain_event_mm",
|
||||
"0x0E": "rain_rate_mm_h",
|
||||
"0x7C": "rain_hourly_mm",
|
||||
"0x10": "rain_daily_mm",
|
||||
"0x11": "rain_weekly_mm",
|
||||
"0x12": "rain_monthly_mm",
|
||||
"0x13": "rain_yearly_mm",
|
||||
}
|
||||
|
||||
def parse(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
received_at: datetime | None = None,
|
||||
) -> WeatherStationObservation:
|
||||
if received_at is None:
|
||||
received_at = datetime.now(timezone.utc)
|
||||
|
||||
common = self._parse_id_list(payload.get("common_list", []))
|
||||
rain = self._parse_id_list(payload.get("rain", []))
|
||||
indoor = self._parse_indoor(payload.get("wh25", []))
|
||||
|
||||
return WeatherStationObservation(
|
||||
observed_at=received_at,
|
||||
received_at=received_at,
|
||||
source="ecowitt_gw3000",
|
||||
temperature_c=common.get("temperature_c"),
|
||||
relative_humidity_pct=common.get("relative_humidity_pct"),
|
||||
dew_point_c=common.get("dew_point_c"),
|
||||
wind_speed_m_s=common.get("wind_speed_m_s"),
|
||||
wind_gust_m_s=common.get("wind_gust_m_s"),
|
||||
wind_max_m_s=common.get("wind_max_m_s"),
|
||||
wind_direction_deg=common.get("wind_direction_deg"),
|
||||
solar_radiation_w_m2=common.get("solar_radiation_w_m2"),
|
||||
uv_index=common.get("uv_index"),
|
||||
rain_rate_mm_h=rain.get("rain_rate_mm_h"),
|
||||
rain_event_mm=rain.get("rain_event_mm"),
|
||||
rain_hourly_mm=rain.get("rain_hourly_mm"),
|
||||
rain_daily_mm=rain.get("rain_daily_mm"),
|
||||
rain_weekly_mm=rain.get("rain_weekly_mm"),
|
||||
rain_monthly_mm=rain.get("rain_monthly_mm"),
|
||||
rain_yearly_mm=rain.get("rain_yearly_mm"),
|
||||
indoor_temperature_c=indoor.get("indoor_temperature_c"),
|
||||
indoor_humidity_pct=indoor.get("indoor_humidity_pct"),
|
||||
absolute_pressure_hpa=indoor.get("absolute_pressure_hpa"),
|
||||
relative_pressure_hpa=indoor.get("relative_pressure_hpa"),
|
||||
metadata={
|
||||
"raw_payload": payload,
|
||||
"debug": payload.get("debug", []),
|
||||
},
|
||||
)
|
||||
|
||||
def _parse_id_list(self, rows: object) -> dict[str, float]:
|
||||
parsed: dict[str, float] = {}
|
||||
if not isinstance(rows, list):
|
||||
return parsed
|
||||
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
row_id = str(row.get("id", ""))
|
||||
key = self.common_id_names.get(row_id) or self.rain_id_names.get(row_id)
|
||||
if key is None:
|
||||
continue
|
||||
value = self._parse_number(row.get("val"))
|
||||
if value is None:
|
||||
continue
|
||||
if key in {"wind_speed_m_s", "wind_gust_m_s", "wind_max_m_s"}:
|
||||
value = self._knots_to_m_s(value)
|
||||
parsed[key] = value
|
||||
|
||||
return parsed
|
||||
|
||||
def _parse_indoor(self, rows: object) -> dict[str, float]:
|
||||
if not isinstance(rows, list) or not rows:
|
||||
return {}
|
||||
row = rows[0]
|
||||
if not isinstance(row, dict):
|
||||
return {}
|
||||
|
||||
values = {
|
||||
"indoor_temperature_c": self._parse_number(row.get("intemp")),
|
||||
"indoor_humidity_pct": self._parse_number(row.get("inhumi")),
|
||||
"absolute_pressure_hpa": self._parse_number(row.get("abs")),
|
||||
"relative_pressure_hpa": self._parse_number(row.get("rel")),
|
||||
}
|
||||
return {
|
||||
key: value
|
||||
for key, value in values.items()
|
||||
if value is not None
|
||||
}
|
||||
|
||||
def _parse_number(self, value: object) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
|
||||
match = re.search(r"-?\d+(?:\.\d+)?", str(value))
|
||||
if match is None:
|
||||
return None
|
||||
return float(match.group(0))
|
||||
|
||||
def _knots_to_m_s(self, value: float) -> float:
|
||||
return value * 0.514444
|
||||
@@ -0,0 +1,254 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import urlopen
|
||||
import json
|
||||
|
||||
from gibil.classes.models import (
|
||||
WeatherForecastPoint,
|
||||
WeatherForecastRun,
|
||||
WeatherResolvedTruth,
|
||||
)
|
||||
|
||||
|
||||
class OpenMeteoClient:
|
||||
"""Fetches external weather forecasts from Open-Meteo."""
|
||||
|
||||
base_url = "https://api.open-meteo.com/v1/forecast"
|
||||
|
||||
def build_url(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
forecast_hours: int = 48,
|
||||
timezone_name: str = "UTC",
|
||||
) -> str:
|
||||
params = {
|
||||
"latitude": latitude,
|
||||
"longitude": longitude,
|
||||
"hourly": ",".join(
|
||||
[
|
||||
"temperature_2m",
|
||||
"shortwave_radiation",
|
||||
"cloud_cover",
|
||||
]
|
||||
),
|
||||
"forecast_hours": forecast_hours,
|
||||
"timezone": timezone_name,
|
||||
}
|
||||
return f"{self.base_url}?{urlencode(params)}"
|
||||
|
||||
def fetch_forecast(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
forecast_hours: int = 48,
|
||||
) -> WeatherForecastRun:
|
||||
url = self.build_url(latitude, longitude, forecast_hours)
|
||||
with urlopen(url, timeout=10) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
|
||||
return OpenMeteoParser().parse_forecast(
|
||||
payload=payload,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
issued_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
class OpenMeteoArchiveClient:
|
||||
"""Fetches historical weather data from Open-Meteo archive."""
|
||||
|
||||
base_url = "https://archive-api.open-meteo.com/v1/archive"
|
||||
|
||||
def build_url(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
start_date: date,
|
||||
end_date: date,
|
||||
timezone_name: str = "UTC",
|
||||
) -> str:
|
||||
params = {
|
||||
"latitude": latitude,
|
||||
"longitude": longitude,
|
||||
"start_date": start_date.isoformat(),
|
||||
"end_date": end_date.isoformat(),
|
||||
"hourly": ",".join(
|
||||
[
|
||||
"temperature_2m",
|
||||
"shortwave_radiation",
|
||||
"cloud_cover",
|
||||
]
|
||||
),
|
||||
"timezone": timezone_name,
|
||||
}
|
||||
return f"{self.base_url}?{urlencode(params)}"
|
||||
|
||||
def fetch_resolved_truth(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
start_date: date,
|
||||
end_date: date,
|
||||
) -> list[WeatherResolvedTruth]:
|
||||
url = self.build_url(latitude, longitude, start_date, end_date)
|
||||
with urlopen(url, timeout=20) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
|
||||
return OpenMeteoArchiveParser().parse_resolved_truth(payload)
|
||||
|
||||
|
||||
class OpenMeteoParser:
|
||||
"""Converts Open-Meteo JSON into clean external forecast records."""
|
||||
|
||||
def parse_forecast(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
issued_at: datetime,
|
||||
) -> WeatherForecastRun:
|
||||
hourly = payload.get("hourly", {})
|
||||
times = hourly.get("time", [])
|
||||
temperatures = hourly.get("temperature_2m", [])
|
||||
radiation = hourly.get("shortwave_radiation", [])
|
||||
cloud_cover = hourly.get("cloud_cover", [])
|
||||
|
||||
points: list[WeatherForecastPoint] = []
|
||||
for index, raw_time in enumerate(times):
|
||||
target_at = self._parse_time(raw_time)
|
||||
horizon_hours = max(
|
||||
0, round((target_at - issued_at).total_seconds() / 3600)
|
||||
)
|
||||
|
||||
points.append(
|
||||
WeatherForecastPoint(
|
||||
issued_at=issued_at,
|
||||
target_at=target_at,
|
||||
horizon_hours=horizon_hours,
|
||||
temperature_c=self._at(temperatures, index),
|
||||
shortwave_radiation_w_m2=self._at(radiation, index),
|
||||
cloud_cover_pct=self._at(cloud_cover, index),
|
||||
)
|
||||
)
|
||||
|
||||
return WeatherForecastRun(
|
||||
issued_at=issued_at,
|
||||
source="open_meteo",
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
points=points,
|
||||
)
|
||||
|
||||
def _parse_time(self, raw_time: str) -> datetime:
|
||||
parsed = datetime.fromisoformat(raw_time)
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
def _at(self, values: list[Any], index: int) -> float | None:
|
||||
if index >= len(values):
|
||||
return None
|
||||
|
||||
value = values[index]
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
return float(value)
|
||||
|
||||
|
||||
class OpenMeteoArchiveParser:
|
||||
"""Converts Open-Meteo archive JSON into resolved truth records."""
|
||||
|
||||
def parse_resolved_truth(self, payload: dict[str, Any]) -> list[WeatherResolvedTruth]:
|
||||
hourly = payload.get("hourly", {})
|
||||
times = hourly.get("time", [])
|
||||
temperatures = hourly.get("temperature_2m", [])
|
||||
radiation = hourly.get("shortwave_radiation", [])
|
||||
cloud_cover = hourly.get("cloud_cover", [])
|
||||
|
||||
truth: list[WeatherResolvedTruth] = []
|
||||
for index, raw_time in enumerate(times):
|
||||
truth.append(
|
||||
WeatherResolvedTruth(
|
||||
resolved_at=self._parse_time(raw_time),
|
||||
temperature_c=self._at(temperatures, index),
|
||||
shortwave_radiation_w_m2=self._at(radiation, index),
|
||||
cloud_cover_pct=self._at(cloud_cover, index),
|
||||
source="open_meteo_archive",
|
||||
)
|
||||
)
|
||||
|
||||
return truth
|
||||
|
||||
def _parse_time(self, raw_time: str) -> datetime:
|
||||
parsed = datetime.fromisoformat(raw_time)
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
def _at(self, values: list[Any], index: int) -> float | None:
|
||||
if index >= len(values):
|
||||
return None
|
||||
|
||||
value = values[index]
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
return float(value)
|
||||
|
||||
|
||||
class WeatherBuilder:
|
||||
"""Builds a clean database-ready set of external weather forecast records."""
|
||||
|
||||
def build_forecast_run(
|
||||
self,
|
||||
source: str,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
points: list[WeatherForecastPoint],
|
||||
issued_at: datetime | None = None,
|
||||
) -> WeatherForecastRun:
|
||||
if issued_at is None:
|
||||
issued_at = datetime.now(timezone.utc)
|
||||
|
||||
clean_points = [
|
||||
WeatherForecastPoint(
|
||||
issued_at=issued_at,
|
||||
target_at=point.target_at,
|
||||
horizon_hours=max(
|
||||
0, round((point.target_at - issued_at).total_seconds() / 3600)
|
||||
),
|
||||
temperature_c=point.temperature_c,
|
||||
shortwave_radiation_w_m2=point.shortwave_radiation_w_m2,
|
||||
cloud_cover_pct=point.cloud_cover_pct,
|
||||
source=source,
|
||||
)
|
||||
for point in sorted(points, key=lambda item: item.target_at)
|
||||
]
|
||||
|
||||
return WeatherForecastRun(
|
||||
issued_at=issued_at,
|
||||
source=source,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
points=clean_points,
|
||||
)
|
||||
|
||||
def points_for_horizon(
|
||||
self,
|
||||
forecast_runs: list[WeatherForecastRun],
|
||||
horizon_hours: int,
|
||||
) -> list[WeatherForecastPoint]:
|
||||
points: list[WeatherForecastPoint] = []
|
||||
for run in forecast_runs:
|
||||
points.extend(
|
||||
point
|
||||
for point in run.points
|
||||
if point.horizon_hours == horizon_hours
|
||||
)
|
||||
|
||||
return sorted(points, key=lambda point: point.target_at)
|
||||
@@ -0,0 +1,640 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from os import environ
|
||||
from typing import Iterator
|
||||
|
||||
from gibil.classes.models import (
|
||||
WeatherForecastPoint,
|
||||
WeatherForecastRun,
|
||||
WeatherResolvedTruth,
|
||||
WeatherStationObservation,
|
||||
)
|
||||
from astrape_oracle.classes.datasets import (
|
||||
WeatherDisplayDataset,
|
||||
WeatherStationDisplayDataset,
|
||||
)
|
||||
|
||||
|
||||
class WeatherStoreConfigurationError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WeatherStoreConfig:
|
||||
database_url: str
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "WeatherStoreConfig":
|
||||
database_url = environ.get("ASTRAPE_DATABASE_URL")
|
||||
if not database_url:
|
||||
raise WeatherStoreConfigurationError(
|
||||
"ASTRAPE_DATABASE_URL is required for weather storage"
|
||||
)
|
||||
|
||||
return cls(database_url=database_url)
|
||||
|
||||
|
||||
class WeatherStore:
|
||||
"""Persists external weather forecasts and resolved truth in TimescaleDB."""
|
||||
|
||||
def __init__(self, config: WeatherStoreConfig) -> None:
|
||||
self.config = config
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "WeatherStore":
|
||||
return cls(WeatherStoreConfig.from_env())
|
||||
|
||||
def initialize(self) -> None:
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("CREATE EXTENSION IF NOT EXISTS timescaledb")
|
||||
cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS weather_forecast_points (
|
||||
issued_at TIMESTAMPTZ NOT NULL,
|
||||
target_at TIMESTAMPTZ NOT NULL,
|
||||
horizon_hours INTEGER NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
latitude DOUBLE PRECISION NOT NULL,
|
||||
longitude DOUBLE PRECISION NOT NULL,
|
||||
temperature_c DOUBLE PRECISION,
|
||||
shortwave_radiation_w_m2 DOUBLE PRECISION,
|
||||
cloud_cover_pct DOUBLE PRECISION,
|
||||
inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (issued_at, target_at, source)
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT create_hypertable(
|
||||
'weather_forecast_points',
|
||||
'target_at',
|
||||
if_not_exists => TRUE
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS weather_resolved_truth (
|
||||
resolved_at TIMESTAMPTZ NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
temperature_c DOUBLE PRECISION,
|
||||
shortwave_radiation_w_m2 DOUBLE PRECISION,
|
||||
cloud_cover_pct DOUBLE PRECISION,
|
||||
inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (resolved_at, source)
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
ALTER TABLE weather_resolved_truth
|
||||
ADD COLUMN IF NOT EXISTS cloud_cover_pct DOUBLE PRECISION
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT create_hypertable(
|
||||
'weather_resolved_truth',
|
||||
'resolved_at',
|
||||
if_not_exists => TRUE
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS weather_station_observations (
|
||||
observed_at TIMESTAMPTZ NOT NULL,
|
||||
received_at TIMESTAMPTZ NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
temperature_c DOUBLE PRECISION,
|
||||
relative_humidity_pct DOUBLE PRECISION,
|
||||
dew_point_c DOUBLE PRECISION,
|
||||
pressure_hpa DOUBLE PRECISION,
|
||||
wind_speed_m_s DOUBLE PRECISION,
|
||||
wind_gust_m_s DOUBLE PRECISION,
|
||||
wind_max_m_s DOUBLE PRECISION,
|
||||
wind_direction_deg DOUBLE PRECISION,
|
||||
solar_radiation_w_m2 DOUBLE PRECISION,
|
||||
uv_index DOUBLE PRECISION,
|
||||
rain_rate_mm_h DOUBLE PRECISION,
|
||||
rain_event_mm DOUBLE PRECISION,
|
||||
rain_hourly_mm DOUBLE PRECISION,
|
||||
rain_daily_mm DOUBLE PRECISION,
|
||||
rain_weekly_mm DOUBLE PRECISION,
|
||||
rain_monthly_mm DOUBLE PRECISION,
|
||||
rain_yearly_mm DOUBLE PRECISION,
|
||||
indoor_temperature_c DOUBLE PRECISION,
|
||||
indoor_humidity_pct DOUBLE PRECISION,
|
||||
absolute_pressure_hpa DOUBLE PRECISION,
|
||||
relative_pressure_hpa DOUBLE PRECISION,
|
||||
metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (observed_at, source)
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT create_hypertable(
|
||||
'weather_station_observations',
|
||||
'observed_at',
|
||||
if_not_exists => TRUE
|
||||
)
|
||||
"""
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
def save_forecast_run(self, forecast_run: WeatherForecastRun) -> int:
|
||||
rows = [
|
||||
(
|
||||
point.issued_at,
|
||||
point.target_at,
|
||||
point.horizon_hours,
|
||||
forecast_run.source,
|
||||
forecast_run.latitude,
|
||||
forecast_run.longitude,
|
||||
point.temperature_c,
|
||||
point.shortwave_radiation_w_m2,
|
||||
point.cloud_cover_pct,
|
||||
)
|
||||
for point in forecast_run.points
|
||||
]
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.executemany(
|
||||
"""
|
||||
INSERT INTO weather_forecast_points (
|
||||
issued_at,
|
||||
target_at,
|
||||
horizon_hours,
|
||||
source,
|
||||
latitude,
|
||||
longitude,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (issued_at, target_at, source)
|
||||
DO UPDATE SET
|
||||
horizon_hours = EXCLUDED.horizon_hours,
|
||||
latitude = EXCLUDED.latitude,
|
||||
longitude = EXCLUDED.longitude,
|
||||
temperature_c = EXCLUDED.temperature_c,
|
||||
shortwave_radiation_w_m2 = EXCLUDED.shortwave_radiation_w_m2,
|
||||
cloud_cover_pct = EXCLUDED.cloud_cover_pct,
|
||||
inserted_at = now()
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
return len(rows)
|
||||
|
||||
def save_resolved_truth(self, truth_points: list[WeatherResolvedTruth]) -> int:
|
||||
rows = [
|
||||
(
|
||||
point.resolved_at,
|
||||
point.source,
|
||||
point.temperature_c,
|
||||
point.shortwave_radiation_w_m2,
|
||||
point.cloud_cover_pct,
|
||||
)
|
||||
for point in truth_points
|
||||
]
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.executemany(
|
||||
"""
|
||||
INSERT INTO weather_resolved_truth (
|
||||
resolved_at,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s)
|
||||
ON CONFLICT (resolved_at, source)
|
||||
DO UPDATE SET
|
||||
temperature_c = EXCLUDED.temperature_c,
|
||||
shortwave_radiation_w_m2 = EXCLUDED.shortwave_radiation_w_m2,
|
||||
cloud_cover_pct = EXCLUDED.cloud_cover_pct,
|
||||
inserted_at = now()
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
return len(rows)
|
||||
|
||||
def save_zero_hour_forecast_as_truth(
|
||||
self, forecast_run: WeatherForecastRun
|
||||
) -> int:
|
||||
truth_points = [
|
||||
WeatherResolvedTruth(
|
||||
resolved_at=point.issued_at,
|
||||
source="open_meteo_zero_hour",
|
||||
temperature_c=point.temperature_c,
|
||||
shortwave_radiation_w_m2=point.shortwave_radiation_w_m2,
|
||||
cloud_cover_pct=point.cloud_cover_pct,
|
||||
)
|
||||
for point in forecast_run.points
|
||||
if point.horizon_hours == 0
|
||||
]
|
||||
return self.save_resolved_truth(truth_points)
|
||||
|
||||
def save_station_observation(
|
||||
self,
|
||||
observation: WeatherStationObservation,
|
||||
) -> int:
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
try:
|
||||
from psycopg.types.json import Jsonb
|
||||
except ImportError as error:
|
||||
raise WeatherStoreConfigurationError(
|
||||
"Install dependencies with `python3 -m pip install -r requirements.txt`"
|
||||
) from error
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO weather_station_observations (
|
||||
observed_at,
|
||||
received_at,
|
||||
source,
|
||||
temperature_c,
|
||||
relative_humidity_pct,
|
||||
dew_point_c,
|
||||
pressure_hpa,
|
||||
wind_speed_m_s,
|
||||
wind_gust_m_s,
|
||||
wind_max_m_s,
|
||||
wind_direction_deg,
|
||||
solar_radiation_w_m2,
|
||||
uv_index,
|
||||
rain_rate_mm_h,
|
||||
rain_event_mm,
|
||||
rain_hourly_mm,
|
||||
rain_daily_mm,
|
||||
rain_weekly_mm,
|
||||
rain_monthly_mm,
|
||||
rain_yearly_mm,
|
||||
indoor_temperature_c,
|
||||
indoor_humidity_pct,
|
||||
absolute_pressure_hpa,
|
||||
relative_pressure_hpa,
|
||||
metadata
|
||||
)
|
||||
VALUES (
|
||||
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
|
||||
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
|
||||
%s, %s, %s, %s, %s
|
||||
)
|
||||
ON CONFLICT (observed_at, source)
|
||||
DO UPDATE SET
|
||||
received_at = EXCLUDED.received_at,
|
||||
temperature_c = EXCLUDED.temperature_c,
|
||||
relative_humidity_pct = EXCLUDED.relative_humidity_pct,
|
||||
dew_point_c = EXCLUDED.dew_point_c,
|
||||
pressure_hpa = EXCLUDED.pressure_hpa,
|
||||
wind_speed_m_s = EXCLUDED.wind_speed_m_s,
|
||||
wind_gust_m_s = EXCLUDED.wind_gust_m_s,
|
||||
wind_max_m_s = EXCLUDED.wind_max_m_s,
|
||||
wind_direction_deg = EXCLUDED.wind_direction_deg,
|
||||
solar_radiation_w_m2 = EXCLUDED.solar_radiation_w_m2,
|
||||
uv_index = EXCLUDED.uv_index,
|
||||
rain_rate_mm_h = EXCLUDED.rain_rate_mm_h,
|
||||
rain_event_mm = EXCLUDED.rain_event_mm,
|
||||
rain_hourly_mm = EXCLUDED.rain_hourly_mm,
|
||||
rain_daily_mm = EXCLUDED.rain_daily_mm,
|
||||
rain_weekly_mm = EXCLUDED.rain_weekly_mm,
|
||||
rain_monthly_mm = EXCLUDED.rain_monthly_mm,
|
||||
rain_yearly_mm = EXCLUDED.rain_yearly_mm,
|
||||
indoor_temperature_c = EXCLUDED.indoor_temperature_c,
|
||||
indoor_humidity_pct = EXCLUDED.indoor_humidity_pct,
|
||||
absolute_pressure_hpa = EXCLUDED.absolute_pressure_hpa,
|
||||
relative_pressure_hpa = EXCLUDED.relative_pressure_hpa,
|
||||
metadata = EXCLUDED.metadata,
|
||||
inserted_at = now()
|
||||
""",
|
||||
(
|
||||
observation.observed_at,
|
||||
observation.received_at,
|
||||
observation.source,
|
||||
observation.temperature_c,
|
||||
observation.relative_humidity_pct,
|
||||
observation.dew_point_c,
|
||||
observation.pressure_hpa,
|
||||
observation.wind_speed_m_s,
|
||||
observation.wind_gust_m_s,
|
||||
observation.wind_max_m_s,
|
||||
observation.wind_direction_deg,
|
||||
observation.solar_radiation_w_m2,
|
||||
observation.uv_index,
|
||||
observation.rain_rate_mm_h,
|
||||
observation.rain_event_mm,
|
||||
observation.rain_hourly_mm,
|
||||
observation.rain_daily_mm,
|
||||
observation.rain_weekly_mm,
|
||||
observation.rain_monthly_mm,
|
||||
observation.rain_yearly_mm,
|
||||
observation.indoor_temperature_c,
|
||||
observation.indoor_humidity_pct,
|
||||
observation.absolute_pressure_hpa,
|
||||
observation.relative_pressure_hpa,
|
||||
Jsonb(observation.metadata),
|
||||
),
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
return 1
|
||||
|
||||
def load_latest_forecast_points(
|
||||
self,
|
||||
start_at: datetime,
|
||||
end_at: datetime,
|
||||
) -> list[WeatherForecastPoint]:
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
issued_at,
|
||||
target_at,
|
||||
horizon_hours,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
FROM (
|
||||
SELECT
|
||||
issued_at,
|
||||
target_at,
|
||||
horizon_hours,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY target_at
|
||||
ORDER BY issued_at DESC
|
||||
) as rn
|
||||
FROM weather_forecast_points
|
||||
WHERE target_at >= %s AND target_at <= %s
|
||||
) as ranked
|
||||
WHERE rn = 1
|
||||
ORDER BY target_at
|
||||
LIMIT 5000
|
||||
""",
|
||||
(start_at, end_at),
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
return [
|
||||
WeatherForecastPoint(
|
||||
issued_at=row[0],
|
||||
target_at=row[1],
|
||||
horizon_hours=row[2],
|
||||
source=row[3],
|
||||
temperature_c=row[4],
|
||||
shortwave_radiation_w_m2=row[5],
|
||||
cloud_cover_pct=row[6],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def load_forecast_history(
|
||||
self,
|
||||
start_at: datetime,
|
||||
end_at: datetime,
|
||||
) -> dict[datetime, list[dict[str, object]]]:
|
||||
"""Every forecast issued for each target in the window, oldest first.
|
||||
|
||||
Unlike `load_latest_forecast_points`, this keeps all vintages of a
|
||||
forecast, so a training example can be shown the forecast that was
|
||||
actually available at the time it was issued rather than the best one
|
||||
known in hindsight.
|
||||
"""
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
issued_at,
|
||||
target_at,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
FROM weather_forecast_points
|
||||
WHERE target_at >= %s
|
||||
AND target_at <= %s
|
||||
ORDER BY target_at, issued_at
|
||||
""",
|
||||
(start_at, end_at),
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
by_target: dict[datetime, list[dict[str, object]]] = {}
|
||||
for row in rows:
|
||||
by_target.setdefault(row[1], []).append(
|
||||
{
|
||||
"issued_at": row[0],
|
||||
"target_at": row[1],
|
||||
"temperature_c": row[2],
|
||||
"shortwave_radiation_w_m2": row[3],
|
||||
"cloud_cover_pct": row[4],
|
||||
}
|
||||
)
|
||||
return by_target
|
||||
|
||||
def load_display_dataset(
|
||||
self,
|
||||
start_at: datetime | None = None,
|
||||
end_at: datetime | None = None,
|
||||
) -> WeatherDisplayDataset:
|
||||
now = datetime.now(timezone.utc)
|
||||
if start_at is None:
|
||||
start_at = now - timedelta(hours=24)
|
||||
if end_at is None:
|
||||
end_at = now + timedelta(hours=48)
|
||||
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
issued_at,
|
||||
target_at,
|
||||
horizon_hours,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
FROM (
|
||||
SELECT
|
||||
issued_at,
|
||||
target_at,
|
||||
horizon_hours,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct,
|
||||
ROW_NUMBER() OVER (PARTITION BY target_at, horizon_hours ORDER BY issued_at DESC) as rn
|
||||
FROM weather_forecast_points
|
||||
WHERE target_at >= %s AND target_at <= %s
|
||||
) as ranked
|
||||
WHERE rn = 1
|
||||
ORDER BY target_at, horizon_hours
|
||||
LIMIT 5000
|
||||
""",
|
||||
(start_at, end_at),
|
||||
)
|
||||
forecast_rows = cursor.fetchall()
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
resolved_at,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
FROM weather_resolved_truth
|
||||
WHERE resolved_at >= %s AND resolved_at <= %s
|
||||
ORDER BY resolved_at
|
||||
LIMIT 5000
|
||||
""",
|
||||
(start_at, now),
|
||||
)
|
||||
truth_rows = cursor.fetchall()
|
||||
|
||||
return WeatherDisplayDataset(
|
||||
forecast_points=[
|
||||
WeatherForecastPoint(
|
||||
issued_at=row[0],
|
||||
target_at=row[1],
|
||||
horizon_hours=row[2],
|
||||
source=row[3],
|
||||
temperature_c=row[4],
|
||||
shortwave_radiation_w_m2=row[5],
|
||||
cloud_cover_pct=row[6],
|
||||
)
|
||||
for row in forecast_rows
|
||||
],
|
||||
resolved_truth=[
|
||||
WeatherResolvedTruth(
|
||||
resolved_at=row[0],
|
||||
source=row[1],
|
||||
temperature_c=row[2],
|
||||
shortwave_radiation_w_m2=row[3],
|
||||
cloud_cover_pct=row[4],
|
||||
)
|
||||
for row in truth_rows
|
||||
],
|
||||
)
|
||||
|
||||
def load_station_display_dataset(
|
||||
self,
|
||||
start_at: datetime | None = None,
|
||||
end_at: datetime | None = None,
|
||||
) -> WeatherStationDisplayDataset:
|
||||
now = datetime.now(timezone.utc)
|
||||
if start_at is None:
|
||||
start_at = now - timedelta(days=7)
|
||||
if end_at is None:
|
||||
end_at = now
|
||||
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
observed_at,
|
||||
received_at,
|
||||
source,
|
||||
temperature_c,
|
||||
relative_humidity_pct,
|
||||
dew_point_c,
|
||||
pressure_hpa,
|
||||
wind_speed_m_s,
|
||||
wind_gust_m_s,
|
||||
wind_max_m_s,
|
||||
wind_direction_deg,
|
||||
solar_radiation_w_m2,
|
||||
uv_index,
|
||||
rain_rate_mm_h,
|
||||
rain_event_mm,
|
||||
rain_hourly_mm,
|
||||
rain_daily_mm,
|
||||
rain_weekly_mm,
|
||||
rain_monthly_mm,
|
||||
rain_yearly_mm,
|
||||
indoor_temperature_c,
|
||||
indoor_humidity_pct,
|
||||
absolute_pressure_hpa,
|
||||
relative_pressure_hpa,
|
||||
metadata
|
||||
FROM weather_station_observations
|
||||
WHERE observed_at >= %s AND observed_at <= %s
|
||||
ORDER BY observed_at
|
||||
LIMIT 100000
|
||||
""",
|
||||
(start_at, end_at),
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
return WeatherStationDisplayDataset(
|
||||
observations=[
|
||||
WeatherStationObservation(
|
||||
observed_at=row[0],
|
||||
received_at=row[1],
|
||||
source=row[2],
|
||||
temperature_c=row[3],
|
||||
relative_humidity_pct=row[4],
|
||||
dew_point_c=row[5],
|
||||
pressure_hpa=row[6],
|
||||
wind_speed_m_s=row[7],
|
||||
wind_gust_m_s=row[8],
|
||||
wind_max_m_s=row[9],
|
||||
wind_direction_deg=row[10],
|
||||
solar_radiation_w_m2=row[11],
|
||||
uv_index=row[12],
|
||||
rain_rate_mm_h=row[13],
|
||||
rain_event_mm=row[14],
|
||||
rain_hourly_mm=row[15],
|
||||
rain_daily_mm=row[16],
|
||||
rain_weekly_mm=row[17],
|
||||
rain_monthly_mm=row[18],
|
||||
rain_yearly_mm=row[19],
|
||||
indoor_temperature_c=row[20],
|
||||
indoor_humidity_pct=row[21],
|
||||
absolute_pressure_hpa=row[22],
|
||||
relative_pressure_hpa=row[23],
|
||||
metadata=row[24] or {},
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def _connection(self) -> Iterator[object]:
|
||||
try:
|
||||
import psycopg
|
||||
except ImportError as error:
|
||||
raise WeatherStoreConfigurationError(
|
||||
"Install dependencies with `python3 -m pip install -r requirements.txt`"
|
||||
) from error
|
||||
|
||||
with psycopg.connect(self.config.database_url) as connection:
|
||||
yield connection
|
||||
Reference in New Issue
Block a user