From 97a27a7f0192925c9f6a658c54335c24a1edb512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20H=2E=20M=C3=B8ller?= Date: Mon, 7 Sep 2026 09:50:33 +0200 Subject: [PATCH 1/3] Add MTG and MTG_pvlib configs --- CHANGELOG.md | 8 ++++++-- config.yaml | 17 +++++++++++++++++ sunflow/forecast.py | 2 ++ sunflow/main.py | 2 +- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42e33ae..a301dc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +### Added - Added `{second}` and `{munite}` as a filename formatting variable [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb -- Use `.expand_dims` instead of `.assign_coords` to make sure we have both time dimension and time coordinates when loading from files [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb - Added a `clearsky` config object which can load clear-sky data from files via `clearsky.path` or generate it with `pvlib` via the `simplified_solis` method [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb -- `check_solar_elevation` now does not assume location is in Copenhagen by default [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb - Added `MIN_SOLAR_ELEVATION_DEGREES` (defaulting to 6 degrees) to configure the minimum maximum-corner solar elevation required to run [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb +- Added configurations to run with MTG data with either calculated or pvlib clearsky [!23](https://github.com/dmidk/sunflow/pull/15), @KristianHMoller +### Changed +- Use `.expand_dims` instead of `.assign_coords` to make sure we have both time dimension and time coordinates when loading from files [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb +- `check_solar_elevation` now does not assume location is in Copenhagen by default [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb ## [v1.2.0] diff --git a/config.yaml b/config.yaml index ee51248..3160beb 100644 --- a/config.yaml +++ b/config.yaml @@ -36,3 +36,20 @@ DWD: # Required: Filename format for input data files # Supports subdirectories via path separators, e.g.: "{year}/{month}/{day}/{dataset_name}_{timestamp}_{domain_satellite_name}.nc" filename_format: "{dataset_name}_{timestamp}_{domain_satellite_name}.nc" + +MTG: + nc_variable_names: + sds: GHI + sds_cs: CSR + filename_format: "GHI/GHI{year}{month}{day}{hour}{minute}.nc" + clearsky: + method: file + path: "CSR/CSR{year}{month}{day}{hour}{minute}.nc" + +MTG_pvlib: + nc_variable_names: + sds: GHI + sds_cs: GHI_clearsky + filename_format: "GHI/GHI{year}{month}{day}{hour}{minute}.nc" + clearsky: + method: pvlib diff --git a/sunflow/forecast.py b/sunflow/forecast.py index 55ddc8a..275dd73 100644 --- a/sunflow/forecast.py +++ b/sunflow/forecast.py @@ -4,6 +4,7 @@ import numpy as np import pvlib import xarray as xr +from loguru import logger from Models.ProbabilisticAdvection import ProbabilisticAdvection from .geospatial import get_coordinates @@ -295,6 +296,7 @@ def compute_ensemble_statistics( statistics: list[str], ) -> dict[str, np.ndarray]: """Compute requested statistics over ensemble members (axis 0).""" + logger.info("Computing ensemble statistics...") computed: dict[str, np.ndarray] = {} for statistic in statistics: match statistic: diff --git a/sunflow/main.py b/sunflow/main.py index 0a4bac9..8f70b14 100644 --- a/sunflow/main.py +++ b/sunflow/main.py @@ -114,7 +114,7 @@ def parse_datetime_with_timezone(datetime_str: str) -> datetime: ) parser.add_argument( "--dataset", - choices=["KNMI", "DWD"], + choices=["KNMI", "DWD", "MTG", "MTG_pvlib"], default="KNMI", help="Choose dataset: Currently, only KNMI and DWD data are supported " "(default: KNMI)", From fec6efc94d1ebf6191105e30636c1e65e189bb91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20H=2E=20M=C3=B8ller?= Date: Mon, 7 Sep 2026 10:26:19 +0200 Subject: [PATCH 2/3] Added the option to provide more configuration parameters to config.yaml --- CHANGELOG.md | 1 + config.yaml | 2 ++ sunflow/config.py | 53 +++++++++++++++++++++++++++++++++++------------ sunflow/main.py | 10 +++++---- 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a301dc2..937c273 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added a `clearsky` config object which can load clear-sky data from files via `clearsky.path` or generate it with `pvlib` via the `simplified_solis` method [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb - Added `MIN_SOLAR_ELEVATION_DEGREES` (defaulting to 6 degrees) to configure the minimum maximum-corner solar elevation required to run [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb - Added configurations to run with MTG data with either calculated or pvlib clearsky [!23](https://github.com/dmidk/sunflow/pull/15), @KristianHMoller +- Added the ability to provide more configuration parameters to config.yaml [!23](https://github.com/dmidk/sunflow/pull/15), @KristianHMoller ### Changed - Use `.expand_dims` instead of `.assign_coords` to make sure we have both time dimension and time coordinates when loading from files [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb diff --git a/config.yaml b/config.yaml index 3160beb..192efd4 100644 --- a/config.yaml +++ b/config.yaml @@ -42,6 +42,7 @@ MTG: sds: GHI sds_cs: CSR filename_format: "GHI/GHI{year}{month}{day}{hour}{minute}.nc" + input_data_frequency_minutes: 10 clearsky: method: file path: "CSR/CSR{year}{month}{day}{hour}{minute}.nc" @@ -51,5 +52,6 @@ MTG_pvlib: sds: GHI sds_cs: GHI_clearsky filename_format: "GHI/GHI{year}{month}{day}{hour}{minute}.nc" + input_data_frequency_minutes: 10 clearsky: method: pvlib diff --git a/sunflow/config.py b/sunflow/config.py index 8eb9db3..cec970f 100644 --- a/sunflow/config.py +++ b/sunflow/config.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import os +from collections.abc import Mapping from dataclasses import dataclass from typing import Self @@ -84,9 +85,16 @@ class NowcastConfig: ensemble_statistics: list[str] @classmethod - def from_env(cls, ensemble_members: int = 1) -> Self: + def from_env( + cls, + ensemble_members: int = 1, + overrides: Mapping[str, object] | None = None, + ) -> Self: """Load nowcast configuration from environment variables with defaults. + Optional ``overrides`` can provide fallback values (for example, from YAML). + Environment variables always take priority over ``overrides``. + Reads the following environment variables: - NOWCAST_DIRECTORY (default: .) @@ -103,6 +111,25 @@ def from_env(cls, ensemble_members: int = 1) -> Self: - ENSEMBLE_STATISTICS (default: median,mean,p10,p25,p75,p90) """ + overrides = overrides or {} + + def _get_value(name: str, default: str) -> str: + env_value = os.getenv(name) + if env_value is not None: + return env_value + + override_value = overrides.get(name.lower()) + if override_value is None: + return default + + if name == "ENSEMBLE_STATISTICS": + if isinstance(override_value, str): + return override_value + if isinstance(override_value, (list, tuple)): + return ",".join(str(item) for item in override_value) + + return str(override_value) + ens_members = ensemble_members # Reference for default noise values: # A. Carpentieri, D. Folini, D. Nerini, S. Pulkkinen, M. Wild, A. Meyer, @@ -112,27 +139,27 @@ def from_env(cls, ensemble_members: int = 1) -> Self: default_alpha = 0.0 if ens_members == 1 else 9.23 default_beta = 0.0 if ens_members == 1 else 0.15 statistics = _parse_ensemble_statistics( - os.getenv("ENSEMBLE_STATISTICS", DEFAULT_ENSEMBLE_STATISTICS) + _get_value("ENSEMBLE_STATISTICS", DEFAULT_ENSEMBLE_STATISTICS) ) return cls( - nowcast_directory=os.getenv("NOWCAST_DIRECTORY", "."), + nowcast_directory=_get_value("NOWCAST_DIRECTORY", "."), ens_members=ens_members, - alpha=float(os.getenv("ALPHA", str(default_alpha))), - beta=float(os.getenv("BETA", str(default_beta))), - past_steps=int(os.getenv("PAST_STEPS", "4")), - future_steps=int(os.getenv("FUTURE_STEPS", "24")), + alpha=float(_get_value("ALPHA", str(default_alpha))), + beta=float(_get_value("BETA", str(default_beta))), + past_steps=int(_get_value("PAST_STEPS", "4")), + future_steps=int(_get_value("FUTURE_STEPS", "24")), input_data_availability_delay_minutes=int( - os.getenv("INPUT_DATA_AVAILABILITY_DELAY_MINUTES", "24") + _get_value("INPUT_DATA_AVAILABILITY_DELAY_MINUTES", "24") ), input_data_frequency_minutes=int( - os.getenv("INPUT_DATA_FREQUENCY_MINUTES", "15") + _get_value("INPUT_DATA_FREQUENCY_MINUTES", "15") ), - max_waiting_time_minutes=int(os.getenv("MAX_WAITING_TIME_MINUTES", "27")), - satellite_data_directory=os.getenv("SATELLITE_DATA_DIRECTORY", "."), - max_clearsky_fallback_days=int(os.getenv("MAX_CLEARSKY_FALLBACK_DAYS", "3")), + max_waiting_time_minutes=int(_get_value("MAX_WAITING_TIME_MINUTES", "27")), + satellite_data_directory=_get_value("SATELLITE_DATA_DIRECTORY", "."), + max_clearsky_fallback_days=int(_get_value("MAX_CLEARSKY_FALLBACK_DAYS", "3")), min_solar_elevation_degrees=float( - os.getenv("MIN_SOLAR_ELEVATION_DEGREES", "6") + _get_value("MIN_SOLAR_ELEVATION_DEGREES", "6") ), ensemble_statistics=statistics, ) diff --git a/sunflow/main.py b/sunflow/main.py index 8f70b14..bfd57ac 100644 --- a/sunflow/main.py +++ b/sunflow/main.py @@ -555,11 +555,15 @@ def cli() -> None: # Load configuration args = parse_arguments() - nowcast_config = NowcastConfig.from_env(ensemble_members=args.ensemble_members) + dataset_name = args.dataset + config = yaml.safe_load(open("config.yaml"))[dataset_name] + nowcast_config = NowcastConfig.from_env( + ensemble_members=args.ensemble_members, + overrides=config, + ) s3_config = S3Config.from_env() run_mode = args.run_mode - dataset_name = args.dataset domain_satellite_name = args.domain_satellite domain_satellite = resolve_domain_bbox( domain_satellite_name, @@ -593,8 +597,6 @@ def cli() -> None: f"domain_nowcast={domain_nowcast}." ) - config = yaml.safe_load(open("config.yaml"))[dataset_name] - if run_mode != "s3": os.makedirs(nowcast_config.nowcast_directory, exist_ok=True) From 4c291d8d0a163ca9ad0aae901a178eeff3d6fccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20H=2E=20M=C3=B8ller?= Date: Mon, 7 Sep 2026 10:37:14 +0200 Subject: [PATCH 3/3] Fixed typo in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 937c273..cb8d288 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Added `{second}` and `{munite}` as a filename formatting variable [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb +- Added `{second}` and `{minute}` as a filename formatting variable [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb - Added a `clearsky` config object which can load clear-sky data from files via `clearsky.path` or generate it with `pvlib` via the `simplified_solis` method [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb - Added `MIN_SOLAR_ELEVATION_DEGREES` (defaulting to 6 degrees) to configure the minimum maximum-corner solar elevation required to run [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb - Added configurations to run with MTG data with either calculated or pvlib clearsky [!23](https://github.com/dmidk/sunflow/pull/15), @KristianHMoller