Add new daemons and debug scripts for Sigenergy and Oracle functionalities
- Implement `sigen_daemon.py` to poll Sigenergy plant metrics and store snapshots. - Create `web_daemon.py` for serving a web interface with various endpoints. - Add debug scripts: - `debug_duplicates.py` to find duplicate target times in forecast data. - `debug_energy_forecast.py` to print baseline energy forecast curves. - `debug_oracle_evaluations.py` to run the oracle evaluator. - `debug_sigen.py` to inspect stored Sigenergy plant snapshots. - `debug_weather.py` to trace resolved truth data. - `modbus_test.py` for exploring Sigenergy plants or inverters over Modbus TCP. - Introduce `oracle_evaluator.py` for evaluating stored oracle predictions against actuals. - Add TCN training scripts in `tcn` directory for training usage sequence models.
This commit is contained in:
rpotter6298
@@ -0,0 +1,23 @@
|
||||
from gibil.classes.weather.builder import (
|
||||
OpenMeteoArchiveClient,
|
||||
OpenMeteoArchiveParser,
|
||||
OpenMeteoClient,
|
||||
OpenMeteoParser,
|
||||
WeatherBuilder,
|
||||
)
|
||||
from gibil.classes.weather.display import WeatherDisplay, WeatherDisplayDataset
|
||||
from gibil.classes.weather.sample_data import WeatherSampleData
|
||||
from gibil.classes.weather.store import WeatherStore, WeatherStoreConfig
|
||||
|
||||
__all__ = [
|
||||
"OpenMeteoClient",
|
||||
"OpenMeteoParser",
|
||||
"OpenMeteoArchiveClient",
|
||||
"OpenMeteoArchiveParser",
|
||||
"WeatherBuilder",
|
||||
"WeatherDisplay",
|
||||
"WeatherDisplayDataset",
|
||||
"WeatherSampleData",
|
||||
"WeatherStore",
|
||||
"WeatherStoreConfig",
|
||||
]
|
||||
@@ -0,0 +1,254 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import urlopen
|
||||
import json
|
||||
|
||||
from gibil.classes.models import (
|
||||
WeatherForecastPoint,
|
||||
WeatherForecastRun,
|
||||
WeatherResolvedTruth,
|
||||
)
|
||||
|
||||
|
||||
class OpenMeteoClient:
|
||||
"""Fetches external weather forecasts from Open-Meteo."""
|
||||
|
||||
base_url = "https://api.open-meteo.com/v1/forecast"
|
||||
|
||||
def build_url(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
forecast_hours: int = 48,
|
||||
timezone_name: str = "UTC",
|
||||
) -> str:
|
||||
params = {
|
||||
"latitude": latitude,
|
||||
"longitude": longitude,
|
||||
"hourly": ",".join(
|
||||
[
|
||||
"temperature_2m",
|
||||
"shortwave_radiation",
|
||||
"cloud_cover",
|
||||
]
|
||||
),
|
||||
"forecast_hours": forecast_hours,
|
||||
"timezone": timezone_name,
|
||||
}
|
||||
return f"{self.base_url}?{urlencode(params)}"
|
||||
|
||||
def fetch_forecast(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
forecast_hours: int = 48,
|
||||
) -> WeatherForecastRun:
|
||||
url = self.build_url(latitude, longitude, forecast_hours)
|
||||
with urlopen(url, timeout=10) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
|
||||
return OpenMeteoParser().parse_forecast(
|
||||
payload=payload,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
issued_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
class OpenMeteoArchiveClient:
|
||||
"""Fetches historical weather data from Open-Meteo archive."""
|
||||
|
||||
base_url = "https://archive-api.open-meteo.com/v1/archive"
|
||||
|
||||
def build_url(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
start_date: date,
|
||||
end_date: date,
|
||||
timezone_name: str = "UTC",
|
||||
) -> str:
|
||||
params = {
|
||||
"latitude": latitude,
|
||||
"longitude": longitude,
|
||||
"start_date": start_date.isoformat(),
|
||||
"end_date": end_date.isoformat(),
|
||||
"hourly": ",".join(
|
||||
[
|
||||
"temperature_2m",
|
||||
"shortwave_radiation",
|
||||
"cloud_cover",
|
||||
]
|
||||
),
|
||||
"timezone": timezone_name,
|
||||
}
|
||||
return f"{self.base_url}?{urlencode(params)}"
|
||||
|
||||
def fetch_resolved_truth(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
start_date: date,
|
||||
end_date: date,
|
||||
) -> list[WeatherResolvedTruth]:
|
||||
url = self.build_url(latitude, longitude, start_date, end_date)
|
||||
with urlopen(url, timeout=20) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
|
||||
return OpenMeteoArchiveParser().parse_resolved_truth(payload)
|
||||
|
||||
|
||||
class OpenMeteoParser:
|
||||
"""Converts Open-Meteo JSON into clean external forecast records."""
|
||||
|
||||
def parse_forecast(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
issued_at: datetime,
|
||||
) -> WeatherForecastRun:
|
||||
hourly = payload.get("hourly", {})
|
||||
times = hourly.get("time", [])
|
||||
temperatures = hourly.get("temperature_2m", [])
|
||||
radiation = hourly.get("shortwave_radiation", [])
|
||||
cloud_cover = hourly.get("cloud_cover", [])
|
||||
|
||||
points: list[WeatherForecastPoint] = []
|
||||
for index, raw_time in enumerate(times):
|
||||
target_at = self._parse_time(raw_time)
|
||||
horizon_hours = max(
|
||||
0, round((target_at - issued_at).total_seconds() / 3600)
|
||||
)
|
||||
|
||||
points.append(
|
||||
WeatherForecastPoint(
|
||||
issued_at=issued_at,
|
||||
target_at=target_at,
|
||||
horizon_hours=horizon_hours,
|
||||
temperature_c=self._at(temperatures, index),
|
||||
shortwave_radiation_w_m2=self._at(radiation, index),
|
||||
cloud_cover_pct=self._at(cloud_cover, index),
|
||||
)
|
||||
)
|
||||
|
||||
return WeatherForecastRun(
|
||||
issued_at=issued_at,
|
||||
source="open_meteo",
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
points=points,
|
||||
)
|
||||
|
||||
def _parse_time(self, raw_time: str) -> datetime:
|
||||
parsed = datetime.fromisoformat(raw_time)
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
def _at(self, values: list[Any], index: int) -> float | None:
|
||||
if index >= len(values):
|
||||
return None
|
||||
|
||||
value = values[index]
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
return float(value)
|
||||
|
||||
|
||||
class OpenMeteoArchiveParser:
|
||||
"""Converts Open-Meteo archive JSON into resolved truth records."""
|
||||
|
||||
def parse_resolved_truth(self, payload: dict[str, Any]) -> list[WeatherResolvedTruth]:
|
||||
hourly = payload.get("hourly", {})
|
||||
times = hourly.get("time", [])
|
||||
temperatures = hourly.get("temperature_2m", [])
|
||||
radiation = hourly.get("shortwave_radiation", [])
|
||||
cloud_cover = hourly.get("cloud_cover", [])
|
||||
|
||||
truth: list[WeatherResolvedTruth] = []
|
||||
for index, raw_time in enumerate(times):
|
||||
truth.append(
|
||||
WeatherResolvedTruth(
|
||||
resolved_at=self._parse_time(raw_time),
|
||||
temperature_c=self._at(temperatures, index),
|
||||
shortwave_radiation_w_m2=self._at(radiation, index),
|
||||
cloud_cover_pct=self._at(cloud_cover, index),
|
||||
source="open_meteo_archive",
|
||||
)
|
||||
)
|
||||
|
||||
return truth
|
||||
|
||||
def _parse_time(self, raw_time: str) -> datetime:
|
||||
parsed = datetime.fromisoformat(raw_time)
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
def _at(self, values: list[Any], index: int) -> float | None:
|
||||
if index >= len(values):
|
||||
return None
|
||||
|
||||
value = values[index]
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
return float(value)
|
||||
|
||||
|
||||
class WeatherBuilder:
|
||||
"""Builds a clean database-ready set of external weather forecast records."""
|
||||
|
||||
def build_forecast_run(
|
||||
self,
|
||||
source: str,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
points: list[WeatherForecastPoint],
|
||||
issued_at: datetime | None = None,
|
||||
) -> WeatherForecastRun:
|
||||
if issued_at is None:
|
||||
issued_at = datetime.now(timezone.utc)
|
||||
|
||||
clean_points = [
|
||||
WeatherForecastPoint(
|
||||
issued_at=issued_at,
|
||||
target_at=point.target_at,
|
||||
horizon_hours=max(
|
||||
0, round((point.target_at - issued_at).total_seconds() / 3600)
|
||||
),
|
||||
temperature_c=point.temperature_c,
|
||||
shortwave_radiation_w_m2=point.shortwave_radiation_w_m2,
|
||||
cloud_cover_pct=point.cloud_cover_pct,
|
||||
source=source,
|
||||
)
|
||||
for point in sorted(points, key=lambda item: item.target_at)
|
||||
]
|
||||
|
||||
return WeatherForecastRun(
|
||||
issued_at=issued_at,
|
||||
source=source,
|
||||
latitude=latitude,
|
||||
longitude=longitude,
|
||||
points=clean_points,
|
||||
)
|
||||
|
||||
def points_for_horizon(
|
||||
self,
|
||||
forecast_runs: list[WeatherForecastRun],
|
||||
horizon_hours: int,
|
||||
) -> list[WeatherForecastPoint]:
|
||||
points: list[WeatherForecastPoint] = []
|
||||
for run in forecast_runs:
|
||||
points.extend(
|
||||
point
|
||||
for point in run.points
|
||||
if point.horizon_hours == horizon_hours
|
||||
)
|
||||
|
||||
return sorted(points, key=lambda point: point.target_at)
|
||||
@@ -0,0 +1,320 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
from gibil.classes.models import WeatherForecastPoint, WeatherResolvedTruth
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WeatherDisplayDataset:
|
||||
forecast_points: list[WeatherForecastPoint]
|
||||
resolved_truth: list[WeatherResolvedTruth]
|
||||
|
||||
|
||||
class WeatherDisplay:
|
||||
"""Renders weather source data for the Astrape web UI."""
|
||||
|
||||
def render(self) -> str:
|
||||
return """
|
||||
<section class="panel weather-panel" data-module="weather-display">
|
||||
<div class="panel-heading">
|
||||
<div>
|
||||
<h2>Weather</h2>
|
||||
<p>External forecast history</p>
|
||||
</div>
|
||||
<div class="control-row">
|
||||
<label>
|
||||
Variable
|
||||
<select id="weather-variable">
|
||||
<option value="temperature_c">Temperature</option>
|
||||
<option value="shortwave_radiation_w_m2">Solar radiation</option>
|
||||
<option value="cloud_cover_pct">Cloud cover</option>
|
||||
</select>
|
||||
</label>
|
||||
<div class="legend-control">
|
||||
<div class="legend-title">Horizons</div>
|
||||
<div id="weather-horizons" class="horizon-options"></div>
|
||||
<div class="horizon-option">
|
||||
<span class="legend-swatch truth-swatch"></span>
|
||||
<span>Resolved truth</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="chart-shell">
|
||||
<canvas id="weather-chart" width="1100" height="420"></canvas>
|
||||
</div>
|
||||
</section>
|
||||
<script>
|
||||
window.astrapeModules = window.astrapeModules || {};
|
||||
window.astrapeModules.weatherDisplay = (() => {
|
||||
const palette = ["#60a5fa", "#34d399", "#fbbf24", "#a78bfa", "#fb7185", "#22d3ee"];
|
||||
const defaultSlots = [
|
||||
{ enabled: true, horizon: 4 },
|
||||
{ enabled: true, horizon: 8 },
|
||||
{ enabled: true, horizon: 12 },
|
||||
{ enabled: false, horizon: 16 },
|
||||
{ enabled: false, horizon: 24 },
|
||||
{ enabled: false, horizon: 36 },
|
||||
];
|
||||
|
||||
function init() {
|
||||
const variable = document.getElementById("weather-variable");
|
||||
variable.addEventListener("change", render);
|
||||
buildHorizonControls();
|
||||
refresh();
|
||||
setInterval(refresh, 5000);
|
||||
}
|
||||
|
||||
async function refresh() {
|
||||
const response = await fetch("/api/weather", { cache: "no-store" });
|
||||
const payload = await response.json();
|
||||
window.astrapeWeatherData = payload;
|
||||
render();
|
||||
}
|
||||
|
||||
function buildHorizonControls() {
|
||||
const horizons = document.getElementById("weather-horizons");
|
||||
horizons.innerHTML = "";
|
||||
const slots = loadSlots();
|
||||
|
||||
slots.forEach((slot, index) => {
|
||||
const option = document.createElement("div");
|
||||
option.className = "horizon-option";
|
||||
option.innerHTML = `
|
||||
<input class="horizon-enabled" type="checkbox" ${slot.enabled ? "checked" : ""}>
|
||||
<span class="legend-swatch" style="background: ${palette[index]}"></span>
|
||||
<input class="horizon-value" type="number" min="1" max="47" step="1" value="${slot.horizon}">
|
||||
<span>h</span>
|
||||
`;
|
||||
const checkbox = option.querySelector(".horizon-enabled");
|
||||
const value = option.querySelector(".horizon-value");
|
||||
checkbox.addEventListener("change", render);
|
||||
value.addEventListener("input", render);
|
||||
horizons.appendChild(option);
|
||||
});
|
||||
}
|
||||
|
||||
function render() {
|
||||
const payload = window.astrapeWeatherData || { forecast_points: [], resolved_truth: [] };
|
||||
const variable = document.getElementById("weather-variable").value;
|
||||
const selectedHorizons = selectedSlots();
|
||||
saveSlots();
|
||||
drawChart(payload, variable, selectedHorizons);
|
||||
}
|
||||
|
||||
function drawChart(payload, variable, selectedHorizons) {
|
||||
const canvas = document.getElementById("weather-chart");
|
||||
const ctx = canvas.getContext("2d");
|
||||
const series = buildSeries(payload, variable, selectedHorizons);
|
||||
|
||||
ctx.clearRect(0, 0, canvas.width, canvas.height);
|
||||
|
||||
const allPoints = series.flatMap((item) => item.points);
|
||||
const windowBounds = oracleAlignedBounds(payload.now);
|
||||
const ys = allPoints.map((point) => point.value).filter((value) => value !== null);
|
||||
if (!ys.length) return;
|
||||
|
||||
const bounds = {
|
||||
minX: windowBounds.minX,
|
||||
maxX: windowBounds.maxX,
|
||||
minY: Math.min(...ys),
|
||||
maxY: Math.max(...ys),
|
||||
};
|
||||
if (bounds.minY === bounds.maxY) {
|
||||
bounds.minY -= 1;
|
||||
bounds.maxY += 1;
|
||||
}
|
||||
|
||||
drawAxes(ctx, canvas, bounds);
|
||||
drawNowMarker(ctx, canvas, bounds, windowBounds.nowX);
|
||||
series.forEach((item) => {
|
||||
drawSeries(ctx, canvas, bounds, item.points, item.color, item.width);
|
||||
});
|
||||
}
|
||||
|
||||
function buildSeries(payload, variable, selectedHorizons) {
|
||||
const series = [];
|
||||
for (const slot of selectedHorizons) {
|
||||
const points = (payload.forecast_points || [])
|
||||
.filter((point) => point.horizon_hours === slot.horizon)
|
||||
.map((point) => ({ target_at: point.target_at, value: point[variable] }))
|
||||
.filter((point) => point.value !== null);
|
||||
series.push({ label: `${slot.horizon}h forecast`, points, color: slot.color, width: 2 });
|
||||
}
|
||||
|
||||
const truth = (payload.resolved_truth || [])
|
||||
.map((point) => ({ target_at: point.resolved_at, value: point[variable] }))
|
||||
.filter((point) => point.value !== null);
|
||||
series.push({ label: "resolved truth", points: truth, color: "#f8fafc", width: 3 });
|
||||
return series;
|
||||
}
|
||||
|
||||
function drawAxes(ctx, canvas, bounds) {
|
||||
const margin = chartMargin();
|
||||
ctx.strokeStyle = "#94a3b8";
|
||||
ctx.lineWidth = 1;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(margin.left, margin.top);
|
||||
ctx.lineTo(margin.left, canvas.height - margin.bottom);
|
||||
ctx.lineTo(canvas.width - margin.right, canvas.height - margin.bottom);
|
||||
ctx.stroke();
|
||||
|
||||
ctx.fillStyle = "#475569";
|
||||
ctx.font = "12px system-ui";
|
||||
ctx.fillText(bounds.maxY.toFixed(1), 10, margin.top + 4);
|
||||
ctx.fillText(bounds.minY.toFixed(1), 10, canvas.height - margin.bottom);
|
||||
}
|
||||
|
||||
function drawSeries(ctx, canvas, bounds, points, color, width) {
|
||||
if (!points.length) return;
|
||||
const margin = chartMargin();
|
||||
if (points.length === 1) {
|
||||
const point = points[0];
|
||||
const x = scale(new Date(point.target_at).getTime(), bounds.minX, bounds.maxX, margin.left, canvas.width - margin.right);
|
||||
const y = scale(point.value, bounds.minY, bounds.maxY, canvas.height - margin.bottom, margin.top);
|
||||
ctx.fillStyle = color;
|
||||
ctx.beginPath();
|
||||
ctx.arc(x, y, width + 3, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
return;
|
||||
}
|
||||
|
||||
ctx.strokeStyle = color;
|
||||
ctx.lineWidth = width;
|
||||
ctx.beginPath();
|
||||
|
||||
points.forEach((point, index) => {
|
||||
const x = scale(new Date(point.target_at).getTime(), bounds.minX, bounds.maxX, margin.left, canvas.width - margin.right);
|
||||
const y = scale(point.value, bounds.minY, bounds.maxY, canvas.height - margin.bottom, margin.top);
|
||||
if (index === 0) ctx.moveTo(x, y);
|
||||
else ctx.lineTo(x, y);
|
||||
});
|
||||
ctx.stroke();
|
||||
}
|
||||
|
||||
function drawNowMarker(ctx, canvas, bounds, now) {
|
||||
if (now < bounds.minX || now > bounds.maxX) return;
|
||||
|
||||
const margin = chartMargin();
|
||||
const x = scale(now, bounds.minX, bounds.maxX, margin.left, canvas.width - margin.right);
|
||||
|
||||
ctx.save();
|
||||
ctx.strokeStyle = "#f8fafc";
|
||||
ctx.lineWidth = 1;
|
||||
ctx.setLineDash([5, 5]);
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(x, margin.top);
|
||||
ctx.lineTo(x, canvas.height - margin.bottom);
|
||||
ctx.stroke();
|
||||
ctx.setLineDash([]);
|
||||
ctx.fillStyle = "#f8fafc";
|
||||
ctx.font = "12px system-ui";
|
||||
ctx.fillText("now", Math.min(x + 8, canvas.width - margin.right - 28), margin.top + 14);
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
function selectedSlots() {
|
||||
return [...document.querySelectorAll("#weather-horizons .horizon-option")]
|
||||
.map((item, index) => {
|
||||
const enabled = item.querySelector(".horizon-enabled").checked;
|
||||
const input = item.querySelector(".horizon-value");
|
||||
const horizon = clamp(Number(input.value), 1, 47);
|
||||
input.value = horizon;
|
||||
return enabled ? { horizon, color: palette[index] } : null;
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function saveSlots() {
|
||||
const slots = [...document.querySelectorAll("#weather-horizons .horizon-option")]
|
||||
.map((item) => ({
|
||||
enabled: item.querySelector(".horizon-enabled").checked,
|
||||
horizon: Number(item.querySelector(".horizon-value").value),
|
||||
}));
|
||||
localStorage.setItem("astrapeWeatherHorizonSlots", JSON.stringify(slots));
|
||||
}
|
||||
|
||||
function loadSlots() {
|
||||
try {
|
||||
const parsed = JSON.parse(localStorage.getItem("astrapeWeatherHorizonSlots"));
|
||||
if (Array.isArray(parsed) && parsed.length === 6) return parsed;
|
||||
} catch (error) {
|
||||
return defaultSlots;
|
||||
}
|
||||
return defaultSlots;
|
||||
}
|
||||
|
||||
function clamp(value, min, max) {
|
||||
if (!Number.isFinite(value)) return min;
|
||||
return Math.min(max, Math.max(min, Math.round(value)));
|
||||
}
|
||||
|
||||
function chartMargin() {
|
||||
return { top: 24, right: 28, bottom: 34, left: 52 };
|
||||
}
|
||||
|
||||
function oracleAlignedBounds(nowIso) {
|
||||
const parsedNow = new Date(nowIso).getTime();
|
||||
const now = Number.isFinite(parsedNow) ? parsedNow : Date.now();
|
||||
return {
|
||||
minX: now - 24 * 60 * 60 * 1000,
|
||||
maxX: now + 48 * 60 * 60 * 1000,
|
||||
nowX: now
|
||||
};
|
||||
}
|
||||
|
||||
function scale(value, inMin, inMax, outMin, outMax) {
|
||||
if (inMin === inMax) return (outMin + outMax) / 2;
|
||||
return outMin + ((value - inMin) / (inMax - inMin)) * (outMax - outMin);
|
||||
}
|
||||
|
||||
return { init };
|
||||
})();
|
||||
window.astrapeModules.weatherDisplay.init();
|
||||
</script>
|
||||
"""
|
||||
|
||||
def data_payload(self, dataset: WeatherDisplayDataset | None = None) -> str:
|
||||
if dataset is None:
|
||||
dataset = WeatherDisplayDataset(forecast_points=[], resolved_truth=[])
|
||||
|
||||
forecast_points = [self._forecast_point(point) for point in dataset.forecast_points]
|
||||
resolved_truth = [self._truth_point(point) for point in dataset.resolved_truth]
|
||||
horizons = sorted({point["horizon_hours"] for point in forecast_points})
|
||||
|
||||
return json.dumps(
|
||||
{
|
||||
"now": datetime.now().astimezone().isoformat(),
|
||||
"forecast_points": forecast_points,
|
||||
"resolved_truth": resolved_truth,
|
||||
"horizons": horizons,
|
||||
"min_horizon": 1,
|
||||
"max_horizon": 47,
|
||||
}
|
||||
)
|
||||
|
||||
def _forecast_point(self, point: WeatherForecastPoint) -> dict[str, object]:
|
||||
return {
|
||||
"issued_at": self._iso(point.issued_at),
|
||||
"target_at": self._iso(point.target_at),
|
||||
"horizon_hours": point.horizon_hours,
|
||||
"source": point.source,
|
||||
"temperature_c": point.temperature_c,
|
||||
"shortwave_radiation_w_m2": point.shortwave_radiation_w_m2,
|
||||
"cloud_cover_pct": point.cloud_cover_pct,
|
||||
}
|
||||
|
||||
def _truth_point(self, point: WeatherResolvedTruth) -> dict[str, object]:
|
||||
return {
|
||||
"resolved_at": self._iso(point.resolved_at),
|
||||
"source": point.source,
|
||||
"temperature_c": point.temperature_c,
|
||||
"shortwave_radiation_w_m2": point.shortwave_radiation_w_m2,
|
||||
"cloud_cover_pct": point.cloud_cover_pct,
|
||||
}
|
||||
|
||||
def _iso(self, value: datetime) -> str:
|
||||
return value.isoformat()
|
||||
@@ -0,0 +1,74 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from math import pi, sin
|
||||
|
||||
from gibil.classes.models import WeatherForecastPoint, WeatherResolvedTruth
|
||||
from gibil.classes.weather.display import WeatherDisplayDataset
|
||||
|
||||
|
||||
class WeatherSampleData:
|
||||
"""Builds temporary display-only weather data for UI tuning."""
|
||||
|
||||
def build(self, hours: int = 72) -> WeatherDisplayDataset:
|
||||
now = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0)
|
||||
start = now - timedelta(hours=hours)
|
||||
horizons = [2, 4, 8, 12, 24]
|
||||
|
||||
forecast_points: list[WeatherForecastPoint] = []
|
||||
resolved_truth: list[WeatherResolvedTruth] = []
|
||||
|
||||
for offset in range(hours + 1):
|
||||
target_at = start + timedelta(hours=offset)
|
||||
truth_temperature = self._temperature(target_at, offset)
|
||||
truth_solar = self._solar(target_at, offset)
|
||||
|
||||
resolved_truth.append(
|
||||
WeatherResolvedTruth(
|
||||
resolved_at=target_at,
|
||||
source="sample",
|
||||
temperature_c=truth_temperature,
|
||||
shortwave_radiation_w_m2=truth_solar,
|
||||
)
|
||||
)
|
||||
|
||||
for horizon in horizons:
|
||||
forecast_points.append(
|
||||
WeatherForecastPoint(
|
||||
issued_at=target_at - timedelta(hours=horizon),
|
||||
target_at=target_at,
|
||||
horizon_hours=horizon,
|
||||
temperature_c=truth_temperature
|
||||
+ self._temperature_error(offset, horizon),
|
||||
shortwave_radiation_w_m2=max(
|
||||
0,
|
||||
truth_solar + self._solar_error(offset, horizon),
|
||||
),
|
||||
cloud_cover_pct=max(
|
||||
0,
|
||||
min(100, 45 + 30 * sin((offset + horizon) / 9)),
|
||||
),
|
||||
source="sample",
|
||||
)
|
||||
)
|
||||
|
||||
return WeatherDisplayDataset(
|
||||
forecast_points=forecast_points,
|
||||
resolved_truth=resolved_truth,
|
||||
)
|
||||
|
||||
def _temperature(self, target_at: datetime, offset: int) -> float:
|
||||
daily = sin(((target_at.hour - 7) / 24) * 2 * pi)
|
||||
slow = sin(offset / 18)
|
||||
return round(6.5 + daily * 5.5 + slow * 1.3, 1)
|
||||
|
||||
def _solar(self, target_at: datetime, offset: int) -> float:
|
||||
daylight = max(0, sin(((target_at.hour - 5) / 15) * pi))
|
||||
cloud_effect = 0.75 + 0.25 * sin(offset / 7)
|
||||
return round(780 * daylight * cloud_effect, 1)
|
||||
|
||||
def _temperature_error(self, offset: int, horizon: int) -> float:
|
||||
return round((horizon / 8) * sin((offset + horizon) / 5), 1)
|
||||
|
||||
def _solar_error(self, offset: int, horizon: int) -> float:
|
||||
return round((horizon * 9) * sin((offset + horizon) / 4), 1)
|
||||
@@ -0,0 +1,357 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from os import environ
|
||||
from typing import Iterator
|
||||
|
||||
from gibil.classes.models import WeatherForecastPoint, WeatherForecastRun, WeatherResolvedTruth
|
||||
from gibil.classes.weather.display import WeatherDisplayDataset
|
||||
|
||||
|
||||
class WeatherStoreConfigurationError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WeatherStoreConfig:
|
||||
database_url: str
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "WeatherStoreConfig":
|
||||
database_url = environ.get("ASTRAPE_DATABASE_URL")
|
||||
if not database_url:
|
||||
raise WeatherStoreConfigurationError(
|
||||
"ASTRAPE_DATABASE_URL is required for weather storage"
|
||||
)
|
||||
|
||||
return cls(database_url=database_url)
|
||||
|
||||
|
||||
class WeatherStore:
|
||||
"""Persists external weather forecasts and resolved truth in TimescaleDB."""
|
||||
|
||||
def __init__(self, config: WeatherStoreConfig) -> None:
|
||||
self.config = config
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "WeatherStore":
|
||||
return cls(WeatherStoreConfig.from_env())
|
||||
|
||||
def initialize(self) -> None:
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("CREATE EXTENSION IF NOT EXISTS timescaledb")
|
||||
cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS weather_forecast_points (
|
||||
issued_at TIMESTAMPTZ NOT NULL,
|
||||
target_at TIMESTAMPTZ NOT NULL,
|
||||
horizon_hours INTEGER NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
latitude DOUBLE PRECISION NOT NULL,
|
||||
longitude DOUBLE PRECISION NOT NULL,
|
||||
temperature_c DOUBLE PRECISION,
|
||||
shortwave_radiation_w_m2 DOUBLE PRECISION,
|
||||
cloud_cover_pct DOUBLE PRECISION,
|
||||
inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (issued_at, target_at, source)
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT create_hypertable(
|
||||
'weather_forecast_points',
|
||||
'target_at',
|
||||
if_not_exists => TRUE
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS weather_resolved_truth (
|
||||
resolved_at TIMESTAMPTZ NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
temperature_c DOUBLE PRECISION,
|
||||
shortwave_radiation_w_m2 DOUBLE PRECISION,
|
||||
cloud_cover_pct DOUBLE PRECISION,
|
||||
inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (resolved_at, source)
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
ALTER TABLE weather_resolved_truth
|
||||
ADD COLUMN IF NOT EXISTS cloud_cover_pct DOUBLE PRECISION
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT create_hypertable(
|
||||
'weather_resolved_truth',
|
||||
'resolved_at',
|
||||
if_not_exists => TRUE
|
||||
)
|
||||
"""
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
def save_forecast_run(self, forecast_run: WeatherForecastRun) -> int:
|
||||
rows = [
|
||||
(
|
||||
point.issued_at,
|
||||
point.target_at,
|
||||
point.horizon_hours,
|
||||
forecast_run.source,
|
||||
forecast_run.latitude,
|
||||
forecast_run.longitude,
|
||||
point.temperature_c,
|
||||
point.shortwave_radiation_w_m2,
|
||||
point.cloud_cover_pct,
|
||||
)
|
||||
for point in forecast_run.points
|
||||
]
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.executemany(
|
||||
"""
|
||||
INSERT INTO weather_forecast_points (
|
||||
issued_at,
|
||||
target_at,
|
||||
horizon_hours,
|
||||
source,
|
||||
latitude,
|
||||
longitude,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (issued_at, target_at, source)
|
||||
DO UPDATE SET
|
||||
horizon_hours = EXCLUDED.horizon_hours,
|
||||
latitude = EXCLUDED.latitude,
|
||||
longitude = EXCLUDED.longitude,
|
||||
temperature_c = EXCLUDED.temperature_c,
|
||||
shortwave_radiation_w_m2 = EXCLUDED.shortwave_radiation_w_m2,
|
||||
cloud_cover_pct = EXCLUDED.cloud_cover_pct,
|
||||
inserted_at = now()
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
return len(rows)
|
||||
|
||||
def save_resolved_truth(self, truth_points: list[WeatherResolvedTruth]) -> int:
|
||||
rows = [
|
||||
(
|
||||
point.resolved_at,
|
||||
point.source,
|
||||
point.temperature_c,
|
||||
point.shortwave_radiation_w_m2,
|
||||
point.cloud_cover_pct,
|
||||
)
|
||||
for point in truth_points
|
||||
]
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.executemany(
|
||||
"""
|
||||
INSERT INTO weather_resolved_truth (
|
||||
resolved_at,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
)
|
||||
VALUES (%s, %s, %s, %s, %s)
|
||||
ON CONFLICT (resolved_at, source)
|
||||
DO UPDATE SET
|
||||
temperature_c = EXCLUDED.temperature_c,
|
||||
shortwave_radiation_w_m2 = EXCLUDED.shortwave_radiation_w_m2,
|
||||
cloud_cover_pct = EXCLUDED.cloud_cover_pct,
|
||||
inserted_at = now()
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
return len(rows)
|
||||
|
||||
def save_zero_hour_forecast_as_truth(
|
||||
self, forecast_run: WeatherForecastRun
|
||||
) -> int:
|
||||
truth_points = [
|
||||
WeatherResolvedTruth(
|
||||
resolved_at=point.issued_at,
|
||||
source="open_meteo_zero_hour",
|
||||
temperature_c=point.temperature_c,
|
||||
shortwave_radiation_w_m2=point.shortwave_radiation_w_m2,
|
||||
cloud_cover_pct=point.cloud_cover_pct,
|
||||
)
|
||||
for point in forecast_run.points
|
||||
if point.horizon_hours == 0
|
||||
]
|
||||
return self.save_resolved_truth(truth_points)
|
||||
|
||||
def load_latest_forecast_points(
|
||||
self,
|
||||
start_at: datetime,
|
||||
end_at: datetime,
|
||||
) -> list[WeatherForecastPoint]:
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
issued_at,
|
||||
target_at,
|
||||
horizon_hours,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
FROM (
|
||||
SELECT
|
||||
issued_at,
|
||||
target_at,
|
||||
horizon_hours,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY target_at
|
||||
ORDER BY issued_at DESC
|
||||
) as rn
|
||||
FROM weather_forecast_points
|
||||
WHERE target_at >= %s AND target_at <= %s
|
||||
) as ranked
|
||||
WHERE rn = 1
|
||||
ORDER BY target_at
|
||||
LIMIT 5000
|
||||
""",
|
||||
(start_at, end_at),
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
return [
|
||||
WeatherForecastPoint(
|
||||
issued_at=row[0],
|
||||
target_at=row[1],
|
||||
horizon_hours=row[2],
|
||||
source=row[3],
|
||||
temperature_c=row[4],
|
||||
shortwave_radiation_w_m2=row[5],
|
||||
cloud_cover_pct=row[6],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def load_display_dataset(
|
||||
self,
|
||||
start_at: datetime | None = None,
|
||||
end_at: datetime | None = None,
|
||||
) -> WeatherDisplayDataset:
|
||||
now = datetime.now(timezone.utc)
|
||||
if start_at is None:
|
||||
start_at = now - timedelta(hours=24)
|
||||
if end_at is None:
|
||||
end_at = now + timedelta(hours=48)
|
||||
|
||||
with self._connection() as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
issued_at,
|
||||
target_at,
|
||||
horizon_hours,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
FROM (
|
||||
SELECT
|
||||
issued_at,
|
||||
target_at,
|
||||
horizon_hours,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct,
|
||||
ROW_NUMBER() OVER (PARTITION BY target_at, horizon_hours ORDER BY issued_at DESC) as rn
|
||||
FROM weather_forecast_points
|
||||
WHERE target_at >= %s AND target_at <= %s
|
||||
) as ranked
|
||||
WHERE rn = 1
|
||||
ORDER BY target_at, horizon_hours
|
||||
LIMIT 5000
|
||||
""",
|
||||
(start_at, end_at),
|
||||
)
|
||||
forecast_rows = cursor.fetchall()
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT
|
||||
resolved_at,
|
||||
source,
|
||||
temperature_c,
|
||||
shortwave_radiation_w_m2,
|
||||
cloud_cover_pct
|
||||
FROM weather_resolved_truth
|
||||
WHERE resolved_at >= %s AND resolved_at <= %s
|
||||
ORDER BY resolved_at
|
||||
LIMIT 5000
|
||||
""",
|
||||
(start_at, now),
|
||||
)
|
||||
truth_rows = cursor.fetchall()
|
||||
|
||||
return WeatherDisplayDataset(
|
||||
forecast_points=[
|
||||
WeatherForecastPoint(
|
||||
issued_at=row[0],
|
||||
target_at=row[1],
|
||||
horizon_hours=row[2],
|
||||
source=row[3],
|
||||
temperature_c=row[4],
|
||||
shortwave_radiation_w_m2=row[5],
|
||||
cloud_cover_pct=row[6],
|
||||
)
|
||||
for row in forecast_rows
|
||||
],
|
||||
resolved_truth=[
|
||||
WeatherResolvedTruth(
|
||||
resolved_at=row[0],
|
||||
source=row[1],
|
||||
temperature_c=row[2],
|
||||
shortwave_radiation_w_m2=row[3],
|
||||
cloud_cover_pct=row[4],
|
||||
)
|
||||
for row in truth_rows
|
||||
],
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def _connection(self) -> Iterator[object]:
|
||||
try:
|
||||
import psycopg
|
||||
except ImportError as error:
|
||||
raise WeatherStoreConfigurationError(
|
||||
"Install dependencies with `python3 -m pip install -r requirements.txt`"
|
||||
) from error
|
||||
|
||||
with psycopg.connect(self.config.database_url) as connection:
|
||||
yield connection
|
||||
Reference in New Issue
Block a user