ryan/Astrape
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add new daemons and debug scripts for Sigenergy and Oracle functionalities

Browse Source 
- Implement `sigen_daemon.py` to poll Sigenergy plant metrics and store snapshots.
- Create `web_daemon.py` for serving a web interface with various endpoints.
- Add debug scripts:
  - `debug_duplicates.py` to find duplicate target times in forecast data.
  - `debug_energy_forecast.py` to print baseline energy forecast curves.
  - `debug_oracle_evaluations.py` to run the oracle evaluator.
  - `debug_sigen.py` to inspect stored Sigenergy plant snapshots.
  - `debug_weather.py` to trace resolved truth data.
  - `modbus_test.py` for exploring Sigenergy plants or inverters over Modbus TCP.
- Introduce `oracle_evaluator.py` for evaluating stored oracle predictions against actuals.
- Add TCN training scripts in `tcn` directory for training usage sequence models.
This commit is contained in:
rpotter6298
2026-04-28 08:14:00 +02:00
parent ff0c65a794
commit c8e3016fd6
55 changed files with 6385 additions and 633 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gibil/classes/predictors/__init__.py
+9
View File
@@ -0,0 +1,9 @@
__all__ = [
"BaselineSolarProductionOracle",
"BaselineUsageOracle",
"DailyUsageOracle",
"HistoricalUsageOracle",
"SequenceUsageOracle",
"NetPowerForecaster",
"RollingSolarRegressionOracle",
]
gibil/classes/predictors/math_utils.py
+69
View File
@@ -0,0 +1,69 @@
from __future__ import annotations
def fit_ridge_regression(
features: list[list[float]],
targets: list[float],
ridge_lambda: float,
) -> list[float] | None:
if not features:
return None
width = len(features[0])
xtx = [[0.0 for _ in range(width)] for _ in range(width)]
xty = [0.0 for _ in range(width)]
for row, target in zip(features, targets):
for i in range(width):
xty[i] += row[i] * target
for j in range(width):
xtx[i][j] += row[i] * row[j]
for i in range(1, width):
xtx[i][i] += ridge_lambda
return solve_linear_system(xtx, xty)
def solve_linear_system(
matrix: list[list[float]],
vector: list[float],
) -> list[float] | None:
size = len(vector)
rows = [matrix[index][:] + [vector[index]] for index in range(size)]
for pivot_index in range(size):
pivot_row = max(
range(pivot_index, size),
key=lambda row_index: abs(rows[row_index][pivot_index]),
)
if abs(rows[pivot_row][pivot_index]) < 1e-9:
return None
rows[pivot_index], rows[pivot_row] = rows[pivot_row], rows[pivot_index]
pivot = rows[pivot_index][pivot_index]
rows[pivot_index] = [value / pivot for value in rows[pivot_index]]
for row_index in range(size):
if row_index == pivot_index:
continue
factor = rows[row_index][pivot_index]
rows[row_index] = [
value - factor * pivot_value
for value, pivot_value in zip(rows[row_index], rows[pivot_index])
]
return [row[-1] for row in rows]
def dot(left: list[float], right: list[float]) -> float:
return sum(left_value * right_value for left_value, right_value in zip(left, right))
def quantile(values: list[float], q: float) -> float:
if not values:
return 0.0
sorted_values = sorted(values)
index = round((len(sorted_values) - 1) * q)
return sorted_values[index]
gibil/classes/predictors/net_forecaster.py
+54
View File
@@ -0,0 +1,54 @@
from __future__ import annotations
from gibil.classes.models import NetPowerForecastPoint, NetPowerForecastRun, PowerForecastRun
class NetPowerForecaster:
"""Combines production and usage curves into expected and interval net power."""
def combine(
self,
solar_run: PowerForecastRun,
load_run: PowerForecastRun,
) -> NetPowerForecastRun:
load_by_target = {point.target_at: point for point in load_run.points}
points: list[NetPowerForecastPoint] = []
for solar_point in solar_run.points:
load_point = load_by_target.get(solar_point.target_at)
if load_point is None:
continue
points.append(
NetPowerForecastPoint(
target_at=solar_point.target_at,
horizon_minutes=solar_point.horizon_minutes,
expected_net_power_w=(
solar_point.p50_power_w - load_point.p50_power_w
),
safe_net_power_w=(
solar_point.p10_power_w - load_point.p90_power_w
),
p10_net_power_w=(
solar_point.p10_power_w - load_point.p90_power_w
),
p50_net_power_w=(
solar_point.p50_power_w - load_point.p50_power_w
),
p90_net_power_w=(
solar_point.p90_power_w - load_point.p10_power_w
),
solar_p50_power_w=solar_point.p50_power_w,
load_p50_power_w=load_point.p50_power_w,
solar_p10_power_w=solar_point.p10_power_w,
solar_p90_power_w=solar_point.p90_power_w,
load_p10_power_w=load_point.p10_power_w,
load_p90_power_w=load_point.p90_power_w,
)
)
return NetPowerForecastRun(
issued_at=solar_run.issued_at,
source="baseline_net_forecaster",
points=points,
)
gibil/classes/predictors/solar_baseline.py
+94
View File
@@ -0,0 +1,94 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from gibil.classes.models import ForecastKind, PowerForecastPoint, PowerForecastRun, WeatherForecastPoint
from gibil.classes.oracle.config import EnergyForecastConfig
from gibil.classes.sigen.store import SigenStore
from gibil.classes.weather.store import WeatherStore
class BaselineSolarProductionOracle:
"""Forecasts solar production from shortwave radiation and recent plant peak."""
model_version = "baseline_solar_radiation_v1"
def __init__(
self,
weather_store: WeatherStore,
sigen_store: SigenStore,
config: EnergyForecastConfig,
) -> None:
self.weather_store = weather_store
self.sigen_store = sigen_store
self.config = config
def forecast(self, issued_at: datetime | None = None) -> PowerForecastRun:
if issued_at is None:
issued_at = datetime.now(timezone.utc)
weather_points = self.weather_store.load_latest_forecast_points(
start_at=issued_at,
end_at=issued_at + timedelta(hours=self.config.horizon_hours),
)
peak_w = self._solar_peak_w()
points = [
self._forecast_point(
weather_point=point,
issued_at=issued_at,
peak_w=peak_w,
)
for point in weather_points
]
return PowerForecastRun(
issued_at=issued_at,
kind=ForecastKind.SOLAR,
source="baseline_solar_oracle",
model_version=self.model_version,
points=points,
)
def _forecast_point(
self,
weather_point: WeatherForecastPoint,
issued_at: datetime,
peak_w: float,
) -> PowerForecastPoint:
radiation = max(weather_point.shortwave_radiation_w_m2 or 0.0, 0.0)
expected = min(peak_w, peak_w * (radiation / 1000.0) * self.config.solar_scale)
cloud_cover = weather_point.cloud_cover_pct
cloud_uncertainty = 1.0
if cloud_cover is not None:
cloud_uncertainty += min(max(cloud_cover, 0.0), 100.0) / 200.0
p10 = max(0.0, expected * (0.75 / cloud_uncertainty))
p90 = min(peak_w, expected * (1.15 * cloud_uncertainty))
return PowerForecastPoint(
target_at=weather_point.target_at,
horizon_minutes=self._horizon_minutes(issued_at, weather_point.target_at),
expected_power_w=expected,
p10_power_w=p10,
p50_power_w=expected,
p90_power_w=p90,
confidence=0.25,
source="open_meteo_shortwave",
model_version=self.model_version,
metadata={
"shortwave_radiation_w_m2": weather_point.shortwave_radiation_w_m2,
"cloud_cover_pct": weather_point.cloud_cover_pct,
"temperature_c": weather_point.temperature_c,
"solar_peak_w": peak_w,
"fallback_reason": "not_enough_solar_training_samples",
},
)
def _solar_peak_w(self) -> float:
recent_peak = self.sigen_store.load_recent_solar_peak_w()
if recent_peak is None or recent_peak <= 0:
return self.config.fallback_solar_peak_w
return recent_peak * max(self.config.solar_peak_headroom, 1.0)
def _horizon_minutes(self, issued_at: datetime, target_at: datetime) -> int:
return max(0, round((target_at - issued_at).total_seconds() / 60))
gibil/classes/predictors/solar_rolling_regression.py
+175
View File
@@ -0,0 +1,175 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from gibil.classes.models import ForecastKind, PowerForecastPoint, PowerForecastRun, WeatherForecastPoint
from gibil.classes.oracle.config import EnergyForecastConfig
from gibil.classes.predictors.solar_baseline import BaselineSolarProductionOracle
from gibil.classes.predictors.math_utils import dot, fit_ridge_regression, quantile
from gibil.classes.sigen.store import SigenStore
from gibil.classes.weather.store import WeatherStore
class RollingSolarRegressionOracle:
"""Forecasts solar production with a rolling ridge regression."""
model_version = "rolling_solar_regression_v1"
def __init__(
self,
weather_store: WeatherStore,
sigen_store: SigenStore,
config: EnergyForecastConfig,
) -> None:
self.weather_store = weather_store
self.sigen_store = sigen_store
self.config = config
def forecast(self, issued_at: datetime | None = None) -> PowerForecastRun:
if issued_at is None:
issued_at = datetime.now(timezone.utc)
weather_points = self.weather_store.load_latest_forecast_points(
start_at=issued_at,
end_at=issued_at + timedelta(hours=self.config.horizon_hours),
)
training_samples = self.sigen_store.load_solar_training_samples(
lookback=timedelta(days=self.config.solar_training_days)
)
model = self._fit_model(training_samples)
if model is None:
return BaselineSolarProductionOracle(
weather_store=self.weather_store,
sigen_store=self.sigen_store,
config=self.config,
).forecast(issued_at=issued_at)
points = [
self._forecast_point(
weather_point=point,
issued_at=issued_at,
model=model,
training_sample_count=len(training_samples),
)
for point in weather_points
]
return PowerForecastRun(
issued_at=issued_at,
kind=ForecastKind.SOLAR,
source="rolling_solar_regression_oracle",
model_version=self.model_version,
points=points,
)
def _fit_model(
self,
samples: list[dict[str, float | int | object]],
) -> "_SolarRegressionModel | None":
if len(samples) < self.config.solar_min_training_samples:
return None
features = [
self._features(
radiation=float(sample["shortwave_radiation_w_m2"]),
cloud_cover=float(sample["cloud_cover_pct"]),
)
for sample in samples
]
targets = [float(sample["solar_power_w"]) for sample in samples]
coefficients = fit_ridge_regression(
features,
targets,
ridge_lambda=self.config.solar_ridge_lambda,
)
if coefficients is None:
return None
residuals = [
target - dot(coefficients, feature)
for feature, target in zip(features, targets)
]
return _SolarRegressionModel(
coefficients=coefficients,
residual_p10=quantile(residuals, 0.10),
residual_p90=quantile(residuals, 0.90),
peak_w=self._solar_peak_w(),
)
def _forecast_point(
self,
weather_point: WeatherForecastPoint,
issued_at: datetime,
model: "_SolarRegressionModel",
training_sample_count: int,
) -> PowerForecastPoint:
radiation = max(weather_point.shortwave_radiation_w_m2 or 0.0, 0.0)
cloud_cover = self._cloud_cover(weather_point.cloud_cover_pct)
expected = model.predict(self._features(radiation, cloud_cover))
expected *= self.config.solar_scale
p10 = max(0.0, expected + model.residual_p10)
p90 = min(model.peak_w, expected + model.residual_p90)
if p90 < expected:
p90 = expected
if p10 > expected:
p10 = expected
return PowerForecastPoint(
target_at=weather_point.target_at,
horizon_minutes=self._horizon_minutes(issued_at, weather_point.target_at),
expected_power_w=expected,
p10_power_w=p10,
p50_power_w=expected,
p90_power_w=p90,
confidence=0.45,
source="rolling_solar_regression",
model_version=self.model_version,
metadata={
"shortwave_radiation_w_m2": weather_point.shortwave_radiation_w_m2,
"cloud_cover_pct": weather_point.cloud_cover_pct,
"temperature_c": weather_point.temperature_c,
"solar_peak_w": model.peak_w,
"training_sample_count": training_sample_count,
"residual_p10_w": model.residual_p10,
"residual_p90_w": model.residual_p90,
},
)
def _features(self, radiation: float, cloud_cover: float) -> list[float]:
radiation_kw = radiation / 1000.0
cloud = cloud_cover / 100.0
clear = 1.0 - cloud
return [
1.0,
radiation_kw,
radiation_kw * clear,
radiation_kw * cloud,
cloud,
]
def _cloud_cover(self, value: float | None) -> float:
if value is None:
return 0.0
return min(max(value, 0.0), 100.0)
def _solar_peak_w(self) -> float:
recent_peak = self.sigen_store.load_recent_solar_peak_w()
if recent_peak is None or recent_peak <= 0:
return self.config.fallback_solar_peak_w
return recent_peak * max(self.config.solar_peak_headroom, 1.0)
def _horizon_minutes(self, issued_at: datetime, target_at: datetime) -> int:
return max(0, round((target_at - issued_at).total_seconds() / 60))
@dataclass(frozen=True)
class _SolarRegressionModel:
coefficients: list[float]
residual_p10: float
residual_p90: float
peak_w: float
def predict(self, features: list[float]) -> float:
return min(max(dot(self.coefficients, features), 0.0), self.peak_w)
gibil/classes/predictors/usage_baseline.py
+76
View File
@@ -0,0 +1,76 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from gibil.classes.models import ForecastKind, PowerForecastPoint, PowerForecastRun
from gibil.classes.oracle.config import EnergyForecastConfig
from gibil.classes.sigen.store import SigenStore
class BaselineUsageOracle:
"""Forecasts near-future load from recent high-resolution Sigen history."""
model_version = "baseline_recent_load_v1"
def __init__(
self,
sigen_store: SigenStore,
config: EnergyForecastConfig,
) -> None:
self.sigen_store = sigen_store
self.config = config
def forecast(
self,
target_times: list[datetime],
issued_at: datetime | None = None,
) -> PowerForecastRun:
if issued_at is None:
issued_at = datetime.now(timezone.utc)
lookback = timedelta(minutes=self.config.load_lookback_minutes)
summary = self.sigen_store.load_recent_power_summary(lookback=lookback)
latest = self.sigen_store.load_latest_snapshot()
fallback_load_w = latest.load_power_w if latest else 0.0
p50 = self._number(summary.get("load_p50_w"), fallback_load_w)
p10 = max(0.0, self._number(summary.get("load_p10_w"), p50 * 0.7))
p90 = max(
self._number(summary.get("load_p90_w"), p50 * 1.5),
p50 * 1.25,
)
points = [
PowerForecastPoint(
target_at=target_at,
horizon_minutes=max(
0, round((target_at - issued_at).total_seconds() / 60)
),
expected_power_w=p50,
p10_power_w=p10,
p50_power_w=p50,
p90_power_w=p90,
confidence=0.35,
source="recent_sigen_load",
model_version=self.model_version,
metadata={
"lookback_minutes": self.config.load_lookback_minutes,
"load_avg_w": summary.get("load_avg_w"),
"load_max_w": summary.get("load_max_w"),
},
)
for target_at in target_times
]
return PowerForecastRun(
issued_at=issued_at,
kind=ForecastKind.LOAD,
source="baseline_usage_oracle",
model_version=self.model_version,
points=points,
)
def _number(self, value: object, fallback: float) -> float:
if value is None:
return float(fallback)
return float(value)
gibil/classes/predictors/usage_daily.py
+188
View File
@@ -0,0 +1,188 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from gibil.classes.models import ForecastKind, PowerForecastPoint, PowerForecastRun
from gibil.classes.oracle.config import EnergyForecastConfig
from gibil.classes.sigen.store import SigenStore
class DailyUsageOracle:
"""Forecasts load from time-of-day history blended with recent load."""
model_version = "daily_usage_profile_v1"
def __init__(
self,
sigen_store: SigenStore,
config: EnergyForecastConfig,
) -> None:
self.sigen_store = sigen_store
self.config = config
def forecast(
self,
target_times: list[datetime],
issued_at: datetime | None = None,
) -> PowerForecastRun:
if issued_at is None:
issued_at = datetime.now(timezone.utc)
recent_summary = self.sigen_store.load_recent_power_summary(
lookback=timedelta(minutes=self.config.load_lookback_minutes)
)
profile = self._daily_profile()
latest = self.sigen_store.load_latest_snapshot()
fallback_load_w = latest.load_power_w if latest else 0.0
recent_p50 = self._number(recent_summary.get("load_p50_w"), fallback_load_w)
recent_p10 = self._number(recent_summary.get("load_p10_w"), recent_p50 * 0.7)
recent_p90 = self._number(recent_summary.get("load_p90_w"), recent_p50 * 1.5)
blend = min(max(self.config.load_recent_blend, 0.0), 1.0)
points = [
self._forecast_point(
target_at=target_at,
issued_at=issued_at,
profile=profile,
recent_p10=recent_p10,
recent_p50=recent_p50,
recent_p90=recent_p90,
blend=blend,
)
for target_at in target_times
]
return PowerForecastRun(
issued_at=issued_at,
kind=ForecastKind.LOAD,
source="daily_usage_oracle",
model_version=self.model_version,
points=points,
)
def _daily_profile(self) -> dict[int, dict[str, float | int]]:
weekly_profile = self.sigen_store.load_load_profile(
lookback=timedelta(days=self.config.load_profile_days),
bucket_minutes=self.config.load_profile_bucket_minutes,
min_samples=self.config.load_profile_min_samples,
timezone_name=self._local_timezone_name(),
)
grouped: dict[int, list[dict[str, float | int]]] = {}
for (_iso_dow, minute_bucket), values in weekly_profile.items():
grouped.setdefault(minute_bucket, []).append(values)
return {
minute_bucket: self._weighted_profile(values)
for minute_bucket, values in grouped.items()
}
def _weighted_profile(
self,
values: list[dict[str, float | int]],
) -> dict[str, float | int]:
total_samples = sum(int(value["sample_count"]) for value in values)
if total_samples <= 0:
total_samples = len(values)
return {
"p10": self._weighted_average(values, "p10", total_samples),
"p50": self._weighted_average(values, "p50", total_samples),
"p90": self._weighted_average(values, "p90", total_samples),
"avg_load_power_w": self._weighted_average(
values,
"avg_load_power_w",
total_samples,
),
"max_load_power_w": max(float(value["max_load_power_w"]) for value in values),
"sample_count": total_samples,
"weekday_bucket_count": len(values),
}
def _weighted_average(
self,
values: list[dict[str, float | int]],
key: str,
total_samples: int,
) -> float:
return sum(
float(value[key]) * int(value["sample_count"])
for value in values
) / total_samples
def _forecast_point(
self,
target_at: datetime,
issued_at: datetime,
profile: dict[int, dict[str, float | int]],
recent_p10: float,
recent_p50: float,
recent_p90: float,
blend: float,
) -> PowerForecastPoint:
profile_key = self._profile_key(target_at)
profile_values = profile.get(profile_key)
if profile_values is None:
p10 = max(0.0, recent_p10)
p50 = max(0.0, recent_p50)
p90 = max(p50 * 1.25, recent_p90)
confidence = 0.25
sample_count = 0
weekday_bucket_count = 0
else:
p10 = self._blend(float(profile_values["p10"]), recent_p10, blend)
p50 = self._blend(float(profile_values["p50"]), recent_p50, blend)
p90 = self._blend(float(profile_values["p90"]), recent_p90, blend)
p10 = max(0.0, min(p10, p50))
p90 = max(p90, p50 * 1.15)
sample_count = int(profile_values["sample_count"])
weekday_bucket_count = int(profile_values["weekday_bucket_count"])
confidence = min(0.65, 0.35 + sample_count / 750.0)
return PowerForecastPoint(
target_at=target_at,
horizon_minutes=max(
0, round((target_at - issued_at).total_seconds() / 60)
),
expected_power_w=p50,
p10_power_w=p10,
p50_power_w=p50,
p90_power_w=p90,
confidence=confidence,
source="time_of_day_load_profile",
model_version=self.model_version,
metadata={
"profile_key": profile_key,
"profile_sample_count": sample_count,
"weekday_bucket_count": weekday_bucket_count,
"recent_blend": blend,
"lookback_days": self.config.load_profile_days,
"bucket_minutes": self.config.load_profile_bucket_minutes,
},
)
def _profile_key(self, target_at: datetime) -> int:
local = target_at.astimezone(self._local_timezone())
minute_of_day = local.hour * 60 + local.minute
return (
minute_of_day // self.config.load_profile_bucket_minutes
) * self.config.load_profile_bucket_minutes
def _local_timezone(self) -> ZoneInfo:
return ZoneInfo(self._local_timezone_name())
def _local_timezone_name(self) -> str:
try:
ZoneInfo(self.config.local_timezone)
except ZoneInfoNotFoundError:
return "UTC"
return self.config.local_timezone
def _blend(self, profile_value: float, recent_value: float, blend: float) -> float:
return profile_value * (1.0 - blend) + recent_value * blend
def _number(self, value: object, fallback: float) -> float:
if value is None:
return float(fallback)
return float(value)
gibil/classes/predictors/usage_historical.py
+142
View File
@@ -0,0 +1,142 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from gibil.classes.models import ForecastKind, PowerForecastPoint, PowerForecastRun
from gibil.classes.oracle.config import EnergyForecastConfig
from gibil.classes.sigen.store import SigenStore
class HistoricalUsageOracle:
"""Forecasts load from time-of-week history blended with recent load."""
model_version = "historical_usage_profile_v1"
def __init__(
self,
sigen_store: SigenStore,
config: EnergyForecastConfig,
) -> None:
self.sigen_store = sigen_store
self.config = config
def forecast(
self,
target_times: list[datetime],
issued_at: datetime | None = None,
) -> PowerForecastRun:
if issued_at is None:
issued_at = datetime.now(timezone.utc)
recent_summary = self.sigen_store.load_recent_power_summary(
lookback=timedelta(minutes=self.config.load_lookback_minutes)
)
profile = self.sigen_store.load_load_profile(
lookback=timedelta(days=self.config.load_profile_days),
bucket_minutes=self.config.load_profile_bucket_minutes,
min_samples=self.config.load_profile_min_samples,
timezone_name=self._local_timezone_name(),
)
latest = self.sigen_store.load_latest_snapshot()
fallback_load_w = latest.load_power_w if latest else 0.0
recent_p50 = self._number(recent_summary.get("load_p50_w"), fallback_load_w)
recent_p10 = self._number(recent_summary.get("load_p10_w"), recent_p50 * 0.7)
recent_p90 = self._number(recent_summary.get("load_p90_w"), recent_p50 * 1.5)
blend = min(max(self.config.load_recent_blend, 0.0), 1.0)
points = [
self._forecast_point(
target_at=target_at,
issued_at=issued_at,
profile=profile,
recent_p10=recent_p10,
recent_p50=recent_p50,
recent_p90=recent_p90,
blend=blend,
)
for target_at in target_times
]
return PowerForecastRun(
issued_at=issued_at,
kind=ForecastKind.LOAD,
source="historical_usage_oracle",
model_version=self.model_version,
points=points,
)
def _forecast_point(
self,
target_at: datetime,
issued_at: datetime,
profile: dict[tuple[int, int], dict[str, float | int]],
recent_p10: float,
recent_p50: float,
recent_p90: float,
blend: float,
) -> PowerForecastPoint:
profile_key = self._profile_key(target_at)
profile_values = profile.get(profile_key)
if profile_values is None:
p10 = max(0.0, recent_p10)
p50 = max(0.0, recent_p50)
p90 = max(p50 * 1.25, recent_p90)
confidence = 0.25
sample_count = 0
else:
p10 = self._blend(float(profile_values["p10"]), recent_p10, blend)
p50 = self._blend(float(profile_values["p50"]), recent_p50, blend)
p90 = self._blend(float(profile_values["p90"]), recent_p90, blend)
p10 = max(0.0, min(p10, p50))
p90 = max(p90, p50 * 1.15)
confidence = min(0.65, 0.35 + float(profile_values["sample_count"]) / 500.0)
sample_count = int(profile_values["sample_count"])
return PowerForecastPoint(
target_at=target_at,
horizon_minutes=max(
0, round((target_at - issued_at).total_seconds() / 60)
),
expected_power_w=p50,
p10_power_w=p10,
p50_power_w=p50,
p90_power_w=p90,
confidence=confidence,
source="time_of_week_load_profile",
model_version=self.model_version,
metadata={
"profile_key": profile_key,
"profile_sample_count": sample_count,
"recent_blend": blend,
"lookback_days": self.config.load_profile_days,
"bucket_minutes": self.config.load_profile_bucket_minutes,
},
)
def _profile_key(self, target_at: datetime) -> tuple[int, int]:
local = target_at.astimezone(self._local_timezone())
minute_of_day = local.hour * 60 + local.minute
bucket = (
minute_of_day // self.config.load_profile_bucket_minutes
) * self.config.load_profile_bucket_minutes
return local.isoweekday(), bucket
def _local_timezone(self) -> ZoneInfo:
return ZoneInfo(self._local_timezone_name())
def _local_timezone_name(self) -> str:
try:
ZoneInfo(self.config.local_timezone)
except ZoneInfoNotFoundError:
return "UTC"
return self.config.local_timezone
def _blend(self, profile_value: float, recent_value: float, blend: float) -> float:
return profile_value * (1.0 - blend) + recent_value * blend
def _number(self, value: object, fallback: float) -> float:
if value is None:
return float(fallback)
return float(value)
gibil/classes/predictors/usage_hybrid_architecture.py
+35
View File
@@ -0,0 +1,35 @@
from __future__ import annotations
from dataclasses import dataclass
from gibil.classes.predictors.usage_sequence_dataset import (
UsageSequenceDatasetBuilder,
UsageSequenceScaleConfig,
)
@dataclass(frozen=True)
class UsageHybridModelShape:
"""Describes the fixed-plus-token sequence model input contract."""
past_scales: tuple[UsageSequenceScaleConfig, ...]
past_fixed_features: tuple[str, ...]
future_fixed_features: tuple[str, ...]
future_steps: int
quantiles: tuple[float, ...] = (0.10, 0.50, 0.90)
@classmethod
def from_dataset_builder(
cls,
builder: UsageSequenceDatasetBuilder,
) -> "UsageHybridModelShape":
return cls(
past_scales=builder.config.past_scales,
past_fixed_features=tuple(builder.past_feature_names),
future_fixed_features=tuple(builder.future_feature_names),
future_steps=builder.future_steps,
)
@property
def output_width(self) -> int:
return self.future_steps * len(self.quantiles)
gibil/classes/predictors/usage_hybrid_tcn.py
+158
View File
@@ -0,0 +1,158 @@
from __future__ import annotations
from dataclasses import dataclass
@dataclass(frozen=True)
class UsageHybridTCNConfig:
past_feature_count: int
future_feature_count: int
future_steps: int
scale_names: tuple[str, ...]
hidden_channels: int = 64
branch_layers: int = 4
dropout: float = 0.10
quantiles: tuple[float, ...] = (0.10, 0.50, 0.90)
def build_usage_hybrid_tcn(config: UsageHybridTCNConfig):
try:
return _build_usage_hybrid_tcn(config)
except ImportError as error:
raise RuntimeError(
"PyTorch is required for TCN training. Install dependencies with "
"`python3 -m pip install -r requirements.txt`."
) from error
def _build_usage_hybrid_tcn(config: UsageHybridTCNConfig):
import torch
from torch import nn
class CausalTrim(nn.Module):
def __init__(self, trim: int) -> None:
super().__init__()
self.trim = trim
def forward(self, value):
if self.trim <= 0:
return value
return value[:, :, :-self.trim]
class TemporalBlock(nn.Module):
def __init__(
self,
in_channels: int,
out_channels: int,
kernel_size: int,
dilation: int,
dropout: float,
) -> None:
super().__init__()
padding = (kernel_size - 1) * dilation
self.net = nn.Sequential(
nn.Conv1d(
in_channels,
out_channels,
kernel_size=kernel_size,
dilation=dilation,
padding=padding,
),
CausalTrim(padding),
nn.ReLU(),
nn.Dropout(dropout),
nn.Conv1d(
out_channels,
out_channels,
kernel_size=kernel_size,
dilation=dilation,
padding=padding,
),
CausalTrim(padding),
nn.ReLU(),
nn.Dropout(dropout),
)
self.residual = (
nn.Conv1d(in_channels, out_channels, kernel_size=1)
if in_channels != out_channels
else nn.Identity()
)
self.activation = nn.ReLU()
def forward(self, value):
return self.activation(self.net(value) + self.residual(value))
class TemporalBranch(nn.Module):
def __init__(self) -> None:
super().__init__()
layers = []
channels = config.past_feature_count
for layer_index in range(config.branch_layers):
layers.append(
TemporalBlock(
in_channels=channels,
out_channels=config.hidden_channels,
kernel_size=5,
dilation=2**layer_index,
dropout=config.dropout,
)
)
channels = config.hidden_channels
self.net = nn.Sequential(*layers)
def forward(self, value):
# Dataset tensors are batch x time x features; Conv1d wants batch x features x time.
encoded = self.net(value.transpose(1, 2))
return encoded[:, :, -1]
class UsageHybridTCN(nn.Module):
def __init__(self) -> None:
super().__init__()
self.branches = nn.ModuleDict(
{name: TemporalBranch() for name in config.scale_names}
)
branch_width = config.hidden_channels * len(config.scale_names)
self.context = nn.Sequential(
nn.Linear(branch_width, config.hidden_channels),
nn.ReLU(),
nn.Dropout(config.dropout),
)
self.future_encoder = nn.Sequential(
nn.Linear(config.future_feature_count, config.hidden_channels),
nn.ReLU(),
)
self.head = nn.Sequential(
nn.Linear(config.hidden_channels * 2, config.hidden_channels),
nn.ReLU(),
nn.Dropout(config.dropout),
nn.Linear(config.hidden_channels, len(config.quantiles)),
)
def forward(self, past_by_scale, future_features):
branch_outputs = [
self.branches[name](past_by_scale[name])
for name in config.scale_names
]
context = self.context(torch.cat(branch_outputs, dim=1))
future = self.future_encoder(future_features)
repeated_context = context.unsqueeze(1).expand(-1, future.size(1), -1)
return self.head(torch.cat([repeated_context, future], dim=2))
return UsageHybridTCN()
def pinball_loss(prediction, target, quantiles: tuple[float, ...]):
try:
import torch
except ImportError as error:
raise RuntimeError(
"PyTorch is required for TCN training. Install dependencies with "
"`python3 -m pip install -r requirements.txt`."
) from error
target = target.unsqueeze(-1)
losses = []
for index, quantile in enumerate(quantiles):
error = target - prediction[:, :, index : index + 1]
losses.append(torch.maximum(quantile * error, (quantile - 1) * error))
return torch.stack(losses, dim=-1).mean()
gibil/classes/predictors/usage_sequence.py
+32
View File
@@ -0,0 +1,32 @@
from __future__ import annotations
from datetime import datetime
from gibil.classes.models import PowerForecastRun
from gibil.classes.oracle.config import EnergyForecastConfig
from gibil.classes.predictors.usage_daily import DailyUsageOracle
from gibil.classes.sigen.store import SigenStore
class SequenceUsageOracle:
"""Forecasts load from recent sequence state when a trained model exists."""
model_version = "sequence_usage_tcn_v1"
def __init__(
self,
sigen_store: SigenStore,
config: EnergyForecastConfig,
) -> None:
self.sigen_store = sigen_store
self.config = config
self.fallback = DailyUsageOracle(sigen_store=sigen_store, config=config)
def forecast(
self,
target_times: list[datetime],
issued_at: datetime | None = None,
) -> PowerForecastRun:
# The sequence model scaffold is present, but production should remain
# deterministic until we have a trained artifact and evaluation history.
return self.fallback.forecast(target_times=target_times, issued_at=issued_at)
gibil/classes/predictors/usage_sequence_dataset.py
+405
View File
@@ -0,0 +1,405 @@
from __future__ import annotations
from bisect import bisect_right
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from math import cos, pi, sin
from os import environ
from typing import Iterator
from gibil.classes.env_loader import EnvLoader
@dataclass(frozen=True)
class UsageSequenceScaleConfig:
name: str
hours: int
step_seconds: int
@dataclass(frozen=True)
class UsageFeatureToken:
name: str
value: float
@dataclass(frozen=True)
class UsageSequenceDatasetConfig:
lookback_days: int = 30
future_hours: int = 24
future_step_minutes: int = 15
stride_minutes: int = 15
local_timezone: str = "Europe/Stockholm"
past_scales: tuple[UsageSequenceScaleConfig, ...] = (
UsageSequenceScaleConfig(name="recent", hours=2, step_seconds=10),
UsageSequenceScaleConfig(name="medium", hours=6, step_seconds=30),
UsageSequenceScaleConfig(name="daily", hours=24, step_seconds=120),
)
@classmethod
def from_env(cls) -> "UsageSequenceDatasetConfig":
EnvLoader().load()
return cls(
lookback_days=int(environ.get("ASTRAPE_USAGE_SEQUENCE_LOOKBACK_DAYS", "30")),
future_hours=int(environ.get("ASTRAPE_USAGE_SEQUENCE_FUTURE_HOURS", "24")),
future_step_minutes=int(
environ.get("ASTRAPE_USAGE_SEQUENCE_FUTURE_STEP_MINUTES", "15")
),
stride_minutes=int(environ.get("ASTRAPE_USAGE_SEQUENCE_STRIDE_MINUTES", "15")),
local_timezone=environ.get(
"ASTRAPE_LOCAL_TIMEZONE",
environ.get("TZ", "Europe/Stockholm"),
),
)
@dataclass(frozen=True)
class UsageSequenceExample:
issued_at: datetime
past_by_scale: dict[str, list[list[float]]]
past_tokens_by_scale: dict[str, list[list[UsageFeatureToken]]]
future_features: list[list[float]]
future_tokens: list[list[UsageFeatureToken]]
targets: list[float]
class UsageSequenceDatasetBuilder:
"""Builds load forecasting windows from Sigen history."""
past_feature_names = [
"load_power_w",
"solar_power_w",
"grid_import_w",
"grid_export_w",
"battery_power_w",
"battery_soc_pct",
"hour_sin",
"hour_cos",
"dow_sin",
"dow_cos",
]
future_feature_names = [
"hour_sin",
"hour_cos",
"dow_sin",
"dow_cos",
"temperature_c",
"shortwave_radiation_w_m2",
"cloud_cover_pct",
]
def __init__(self, config: UsageSequenceDatasetConfig) -> None:
self.config = config
@classmethod
def from_env(cls) -> "UsageSequenceDatasetBuilder":
return cls(UsageSequenceDatasetConfig.from_env())
def build(self, limit: int | None = None) -> list[UsageSequenceExample]:
samples_by_scale = {
scale.name: self._load_samples(step_seconds=scale.step_seconds)
for scale in self.config.past_scales
}
target_samples = self._load_samples(
step_seconds=self.config.future_step_minutes * 60
)
weather_by_target = self._load_weather_forecasts()
if not target_samples or any(not samples for samples in samples_by_scale.values()):
return []
by_scale = {
name: {sample["bucket"]: sample for sample in samples}
for name, samples in samples_by_scale.items()
}
target_by_time = {
sample["bucket"]: sample
for sample in target_samples
}
first_available = max(samples[0]["bucket"] for samples in samples_by_scale.values())
last_available = min(
[samples[-1]["bucket"] for samples in samples_by_scale.values()]
+ [target_samples[-1]["bucket"]]
)
start_at = first_available + timedelta(hours=self.max_past_hours)
end_at = last_available - timedelta(hours=self.config.future_hours)
issued_at = self._ceil_time(start_at, self.config.stride_minutes)
examples: list[UsageSequenceExample] = []
while issued_at <= end_at:
example = self._build_example(
issued_at,
by_scale,
target_by_time,
weather_by_target,
)
if example is not None:
examples.append(example)
if limit is not None and len(examples) >= limit:
break
issued_at += timedelta(minutes=self.config.stride_minutes)
return examples
def iter_examples(self) -> Iterator[UsageSequenceExample]:
for example in self.build():
yield example
def _build_example(
self,
issued_at: datetime,
by_scale: dict[str, dict[datetime, dict[str, object]]],
target_by_time: dict[datetime, dict[str, object]],
weather_by_target: dict[datetime, list[dict[str, object]]],
) -> UsageSequenceExample | None:
future_times = [
issued_at + timedelta(minutes=self.config.future_step_minutes * offset)
for offset in range(1, self.future_steps + 1)
]
past_by_scale: dict[str, list[list[float]]] = {}
past_tokens_by_scale: dict[str, list[list[UsageFeatureToken]]] = {}
for scale in self.config.past_scales:
past_times = [
issued_at - timedelta(seconds=scale.step_seconds * offset)
for offset in range(self.past_steps(scale), 0, -1)
]
past_rows = [
by_scale[scale.name].get(target_at)
for target_at in past_times
]
if any(row is None or row["load_power_w"] is None for row in past_rows):
return None
past_by_scale[scale.name] = [
self._past_features(row) for row in past_rows if row is not None
]
past_tokens_by_scale[scale.name] = [
self._past_tokens(row) for row in past_rows if row is not None
]
future_rows = [target_by_time.get(target_at) for target_at in future_times]
if any(row is None or row["load_power_w"] is None for row in future_rows):
return None
return UsageSequenceExample(
issued_at=issued_at,
past_by_scale=past_by_scale,
past_tokens_by_scale=past_tokens_by_scale,
future_features=[
self._future_features(target_at, issued_at, weather_by_target)
for target_at in future_times
],
future_tokens=[
self._future_tokens(target_at=target_at, issued_at=issued_at)
for target_at in future_times
],
targets=[
float(row["load_power_w"])
for row in future_rows
if row is not None
],
)
@property
def max_past_hours(self) -> int:
return max(scale.hours for scale in self.config.past_scales)
def past_steps(self, scale: UsageSequenceScaleConfig) -> int:
return scale.hours * 60 * 60 // scale.step_seconds
@property
def future_steps(self) -> int:
return self.config.future_hours * 60 // self.config.future_step_minutes
def _past_features(self, row: dict[str, object]) -> list[float]:
time_features = self._time_features(row["bucket"])
return [
self._number(row["load_power_w"]),
self._number(row["solar_power_w"]),
self._number(row["grid_import_w"]),
self._number(row["grid_export_w"]),
self._number(row["battery_power_w"]),
self._number(row["battery_soc_pct"]),
*time_features,
]
def _past_tokens(self, row: dict[str, object]) -> list[UsageFeatureToken]:
return []
def _time_features(self, value: object) -> list[float]:
timestamp = value
if not isinstance(timestamp, datetime):
raise TypeError("timestamp must be a datetime")
local = timestamp.astimezone(timezone.utc)
minutes = local.hour * 60 + local.minute
minute_angle = 2 * pi * minutes / 1440
dow_angle = 2 * pi * (local.isoweekday() - 1) / 7
return [
sin(minute_angle),
cos(minute_angle),
sin(dow_angle),
cos(dow_angle),
]
def _future_features(
self,
target_at: datetime,
issued_at: datetime,
weather_by_target: dict[datetime, list[dict[str, object]]],
) -> list[float]:
weather = self._weather_for_target(
target_at=target_at,
issued_at=issued_at,
weather_by_target=weather_by_target,
)
return [
*self._time_features(target_at),
self._number(weather.get("temperature_c")),
self._number(weather.get("shortwave_radiation_w_m2")),
self._number(weather.get("cloud_cover_pct")),
]
def _future_tokens(
self,
target_at: datetime,
issued_at: datetime,
) -> list[UsageFeatureToken]:
return []
def _weather_for_target(
self,
target_at: datetime,
issued_at: datetime,
weather_by_target: dict[datetime, list[dict[str, object]]],
) -> dict[str, object]:
forecast_target_at = self._floor_time(target_at, step_minutes=60)
rows = weather_by_target.get(forecast_target_at, [])
if not rows:
return {}
issued_values = [row["issued_at"] for row in rows]
index = bisect_right(issued_values, issued_at) - 1
if index < 0:
return {}
return rows[index]
def _load_samples(self, step_seconds: int) -> list[dict[str, object]]:
EnvLoader().load()
database_url = environ.get("ASTRAPE_DATABASE_URL")
if not database_url:
raise RuntimeError("ASTRAPE_DATABASE_URL is required")
start_at = datetime.now(timezone.utc) - timedelta(days=self.config.lookback_days)
bucket = self._bucket_interval(step_seconds)
try:
import psycopg
except ImportError as error:
raise RuntimeError(
"Install dependencies with `python3 -m pip install -r requirements.txt`"
) from error
with psycopg.connect(database_url) as connection:
with connection.cursor() as cursor:
cursor.execute(
f"""
SELECT
time_bucket('{bucket}', observed_at) AS bucket,
avg(load_power_w) AS load_power_w,
avg(solar_power_w) AS solar_power_w,
avg(grid_import_w) AS grid_import_w,
avg(grid_export_w) AS grid_export_w,
avg(battery_power_w) AS battery_power_w,
avg(battery_soc_pct) AS battery_soc_pct
FROM sigen_plant_snapshots
WHERE observed_at >= %s
AND observed_at <= now()
GROUP BY bucket
ORDER BY bucket
""",
(start_at,),
)
rows = cursor.fetchall()
return [
{
"bucket": row[0],
"load_power_w": row[1],
"solar_power_w": row[2],
"grid_import_w": row[3],
"grid_export_w": row[4],
"battery_power_w": row[5],
"battery_soc_pct": row[6],
}
for row in rows
]
def _load_weather_forecasts(self) -> dict[datetime, list[dict[str, object]]]:
EnvLoader().load()
database_url = environ.get("ASTRAPE_DATABASE_URL")
if not database_url:
raise RuntimeError("ASTRAPE_DATABASE_URL is required")
start_at = datetime.now(timezone.utc) - timedelta(days=self.config.lookback_days)
end_at = datetime.now(timezone.utc) + timedelta(hours=self.config.future_hours)
try:
import psycopg
except ImportError as error:
raise RuntimeError(
"Install dependencies with `python3 -m pip install -r requirements.txt`"
) from error
with psycopg.connect(database_url) as connection:
with connection.cursor() as cursor:
cursor.execute(
"""
SELECT
issued_at,
target_at,
temperature_c,
shortwave_radiation_w_m2,
cloud_cover_pct
FROM weather_forecast_points
WHERE target_at >= %s
AND target_at <= %s
ORDER BY target_at, issued_at
""",
(start_at, end_at),
)
rows = cursor.fetchall()
by_target: dict[datetime, list[dict[str, object]]] = {}
for row in rows:
by_target.setdefault(row[1], []).append(
{
"issued_at": row[0],
"target_at": row[1],
"temperature_c": row[2],
"shortwave_radiation_w_m2": row[3],
"cloud_cover_pct": row[4],
}
)
return by_target
def _bucket_interval(self, step_seconds: int) -> str:
if step_seconds % 60 == 0:
return f"{step_seconds // 60} minutes"
return f"{step_seconds} seconds"
def _ceil_time(self, value: datetime, step_minutes: int) -> datetime:
step_seconds = step_minutes * 60
timestamp = value.timestamp()
remainder = timestamp % step_seconds
if remainder:
timestamp += step_seconds - remainder
return datetime.fromtimestamp(timestamp, timezone.utc)
def _floor_time(self, value: datetime, step_minutes: int) -> datetime:
step_seconds = step_minutes * 60
timestamp = value.timestamp()
timestamp -= timestamp % step_seconds
return datetime.fromtimestamp(timestamp, timezone.utc)
def _number(self, value: object) -> float:
if value is None:
return 0.0
return float(value)