Add MTG and MTG_pvlib configs

CHANGELOG.md

@@ -6,12 +6,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
-- Added `{second}` and `{munite}` as a filename formatting variable [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb
-- Use `.expand_dims` instead of `.assign_coords` to make sure we have both time dimension and time coordinates when loading from files [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb
+
+### Added
+- Added `{second}` and `{minute}` as a filename formatting variable [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb
 - Added a `clearsky` config object which can load clear-sky data from files via `clearsky.path` or generate it with `pvlib` via the `simplified_solis` method [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb
-- `check_solar_elevation` now does not assume location is in Copenhagen by default [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb
 - Added `MIN_SOLAR_ELEVATION_DEGREES` (defaulting to 6 degrees) to configure the minimum maximum-corner solar elevation required to run [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb
+- Added configurations to run with MTG data with either calculated or pvlib clearsky [!23](https://github.com/dmidk/sunflow/pull/15), @KristianHMoller
+- Added the ability to provide more configuration parameters to config.yaml [!23](https://github.com/dmidk/sunflow/pull/15), @KristianHMoller
 
+### Changed
+- Use `.expand_dims` instead of `.assign_coords` to make sure we have both time dimension and time coordinates when loading from files [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb
+- `check_solar_elevation` now does not assume location is in Copenhagen by default [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb
 
 ## [v1.2.0]
 

config.yaml

@@ -36,3 +36,22 @@ DWD:
   # Required: Filename format for input data files
   # Supports subdirectories via path separators, e.g.: "{year}/{month}/{day}/{dataset_name}_{timestamp}_{domain_satellite_name}.nc"
   filename_format: "{dataset_name}_{timestamp}_{domain_satellite_name}.nc"
+
+MTG:
+  nc_variable_names:
+    sds: GHI
+    sds_cs: CSR
+  filename_format: "GHI/GHI{year}{month}{day}{hour}{minute}.nc"
+  input_data_frequency_minutes: 10
+  clearsky:
+    method: file
+    path: "CSR/CSR{year}{month}{day}{hour}{minute}.nc"
+
+MTG_pvlib:
+  nc_variable_names:
+    sds: GHI
+    sds_cs: GHI_clearsky
+  filename_format: "GHI/GHI{year}{month}{day}{hour}{minute}.nc"
+  input_data_frequency_minutes: 10
+  clearsky:
+    method: pvlib

sunflow/config.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import os
+from collections.abc import Mapping
 from dataclasses import dataclass
 from typing import Self
 
@@ -84,9 +85,16 @@ class NowcastConfig:
     ensemble_statistics: list[str]
 
     @classmethod
-    def from_env(cls, ensemble_members: int = 1) -> Self:
+    def from_env(
+        cls,
+        ensemble_members: int = 1,
+        overrides: Mapping[str, object] | None = None,
+    ) -> Self:
         """Load nowcast configuration from environment variables with defaults.
 
+        Optional ``overrides`` can provide fallback values (for example, from YAML).
+        Environment variables always take priority over ``overrides``.
+
         Reads the following environment variables:
 
         - NOWCAST_DIRECTORY (default: .)
@@ -103,6 +111,25 @@ def from_env(cls, ensemble_members: int = 1) -> Self:
         - ENSEMBLE_STATISTICS (default: median,mean,p10,p25,p75,p90)
         """
 
+        overrides = overrides or {}
+
+        def _get_value(name: str, default: str) -> str:
+            env_value = os.getenv(name)
+            if env_value is not None:
+                return env_value
+
+            override_value = overrides.get(name.lower())
+            if override_value is None:
+                return default
+
+            if name == "ENSEMBLE_STATISTICS":
+                if isinstance(override_value, str):
+                    return override_value
+                if isinstance(override_value, (list, tuple)):
+                    return ",".join(str(item) for item in override_value)
+
+            return str(override_value)
+
         ens_members = ensemble_members
         # Reference for default noise values:
         # A. Carpentieri, D. Folini, D. Nerini, S. Pulkkinen, M. Wild, A. Meyer,
@@ -112,27 +139,27 @@ def from_env(cls, ensemble_members: int = 1) -> Self:
         default_alpha = 0.0 if ens_members == 1 else 9.23
         default_beta = 0.0 if ens_members == 1 else 0.15
         statistics = _parse_ensemble_statistics(
-            os.getenv("ENSEMBLE_STATISTICS", DEFAULT_ENSEMBLE_STATISTICS)
+            _get_value("ENSEMBLE_STATISTICS", DEFAULT_ENSEMBLE_STATISTICS)
         )
 
         return cls(
-            nowcast_directory=os.getenv("NOWCAST_DIRECTORY", "."),
+            nowcast_directory=_get_value("NOWCAST_DIRECTORY", "."),
             ens_members=ens_members,
-            alpha=float(os.getenv("ALPHA", str(default_alpha))),
-            beta=float(os.getenv("BETA", str(default_beta))),
-            past_steps=int(os.getenv("PAST_STEPS", "4")),
-            future_steps=int(os.getenv("FUTURE_STEPS", "24")),
+            alpha=float(_get_value("ALPHA", str(default_alpha))),
+            beta=float(_get_value("BETA", str(default_beta))),
+            past_steps=int(_get_value("PAST_STEPS", "4")),
+            future_steps=int(_get_value("FUTURE_STEPS", "24")),
             input_data_availability_delay_minutes=int(
-                os.getenv("INPUT_DATA_AVAILABILITY_DELAY_MINUTES", "24")
+                _get_value("INPUT_DATA_AVAILABILITY_DELAY_MINUTES", "24")
             ),
             input_data_frequency_minutes=int(
-                os.getenv("INPUT_DATA_FREQUENCY_MINUTES", "15")
+                _get_value("INPUT_DATA_FREQUENCY_MINUTES", "15")
             ),
-            max_waiting_time_minutes=int(os.getenv("MAX_WAITING_TIME_MINUTES", "27")),
-            satellite_data_directory=os.getenv("SATELLITE_DATA_DIRECTORY", "."),
-            max_clearsky_fallback_days=int(os.getenv("MAX_CLEARSKY_FALLBACK_DAYS", "3")),
+            max_waiting_time_minutes=int(_get_value("MAX_WAITING_TIME_MINUTES", "27")),
+            satellite_data_directory=_get_value("SATELLITE_DATA_DIRECTORY", "."),
+            max_clearsky_fallback_days=int(_get_value("MAX_CLEARSKY_FALLBACK_DAYS", "3")),
             min_solar_elevation_degrees=float(
-                os.getenv("MIN_SOLAR_ELEVATION_DEGREES", "6")
+                _get_value("MIN_SOLAR_ELEVATION_DEGREES", "6")
             ),
             ensemble_statistics=statistics,
         )

sunflow/forecast.py

@@ -4,6 +4,7 @@
 import numpy as np
 import pvlib
 import xarray as xr
+from loguru import logger
 from Models.ProbabilisticAdvection import ProbabilisticAdvection
 
 from .geospatial import get_coordinates
@@ -295,6 +296,7 @@ def compute_ensemble_statistics(
     statistics: list[str],
 ) -> dict[str, np.ndarray]:
     """Compute requested statistics over ensemble members (axis 0)."""
+    logger.info("Computing ensemble statistics...")
     computed: dict[str, np.ndarray] = {}
     for statistic in statistics:
         match statistic:

sunflow/main.py

@@ -114,7 +114,7 @@ def parse_datetime_with_timezone(datetime_str: str) -> datetime:
     )
     parser.add_argument(
         "--dataset",
-        choices=["KNMI", "DWD"],
+        choices=["KNMI", "DWD", "MTG", "MTG_pvlib"],
         default="KNMI",
         help="Choose dataset: Currently, only KNMI and DWD data are supported "
         "(default: KNMI)",
@@ -555,11 +555,15 @@ def cli() -> None:
 
     # Load configuration
     args = parse_arguments()
-    nowcast_config = NowcastConfig.from_env(ensemble_members=args.ensemble_members)
+    dataset_name = args.dataset
+    config = yaml.safe_load(open("config.yaml"))[dataset_name]
+    nowcast_config = NowcastConfig.from_env(
+        ensemble_members=args.ensemble_members,
+        overrides=config,
+    )
     s3_config = S3Config.from_env()
 
     run_mode = args.run_mode
-    dataset_name = args.dataset
     domain_satellite_name = args.domain_satellite
     domain_satellite = resolve_domain_bbox(
         domain_satellite_name,
@@ -593,8 +597,6 @@ def cli() -> None:
             f"domain_nowcast={domain_nowcast}."
         )
 
-    config = yaml.safe_load(open("config.yaml"))[dataset_name]
-
     if run_mode != "s3":
         os.makedirs(nowcast_config.nowcast_directory, exist_ok=True)