RHIDP-18071: extend environment variable parsing behaviour

docs/user_doc/deployment_guide.md

@@ -4,6 +4,8 @@
 
 * [Preface](#preface)
 * [Deployment methods](#deployment-methods)
+* [Configuration modes](#configuration-modes)
+    * [Environment variable references](#environment-variable-references)
 * [Integration with OGX framework](#integration-with-ogx-framework)
     * [OGX as a library](#ogx-as-a-library)
     * [OGX as a server](#ogx-as-a-server)
@@ -108,6 +110,38 @@ fails if a unified synthesis input and `library_client_config_path` are both
 present.
 
 
+### Environment variable references
+
+`lightspeed-stack.yaml` accepts `${VAR}` wherever `${env.VAR}` is supported.
+Both forms behave the same. Variable names contain one or more uppercase
+letters, digits, or underscores.
+
+Both spellings support these forms:
+
+- `${VAR}` or `${env.VAR}` uses the variable's value. Resolution fails if the
+  variable is missing or empty.
+- `${VAR:=default}` or `${env.VAR:=default}` uses `default` when the variable
+  is missing or empty.
+- `${VAR:+value}` or `${env.VAR:+value}` uses `value` when the variable is
+  non-empty. Otherwise, it substitutes an empty string.
+
+The shell-style `:-` default operator is not supported. OGX converts resolved
+numeric strings to numbers and boolean strings to booleans. An empty result
+becomes `null`.
+
+Both references below use defaults if their variables are missing or empty:
+
+```yaml
+service:
+  host: "${SERVICE_HOST:=0.0.0.0}"
+  port: "${env.SERVICE_PORT:=8080}"
+```
+
+LCORE resolves its configuration values while loading. For deferred values,
+including those in `ogx.config.native_override`, LCORE writes `${env.VAR}`
+references to the generated `run.yaml`. OGX resolves them at startup.
+
+
 
 ## Integration with OGX framework
 
@@ -283,7 +317,7 @@ llama_stack:
    ```
 
 3. Replace literal secrets. If your `run.yaml` contained secret values
-   directly, replace them with `${env.MY_VAR}` environment references —
+   directly, replace them with `${MY_VAR}` or `${env.MY_VAR}` environment references —
    the migrated file otherwise carries them onto disk verbatim (the
    synthesized output is written owner-only, mode 0600, as a safety net).
 

scripts/generate_openapi_schema.py

@@ -70,6 +70,7 @@ def generate_openapi_schema(schema_file_name: str) -> None:
         AssertionError: If the schema and project versions differ.
         subprocess.CalledProcessError: If ``pdm show --version`` fails.
         OSError: If the file cannot be written or PDM cannot be started.
+
     """
     print("Service metadata:")
     print(app.title)

scripts/latest_tag.py

@@ -34,6 +34,7 @@ def get_latest_stable() -> Optional[str]:
     Raises:
         subprocess.CalledProcessError: If ``git tag`` fails.
         OSError: If Git cannot be started.
+
     """
     stdout = subprocess.check_output(["git", "tag"], text=True)
     tags = [tag for tag in stdout.splitlines() if not is_prerelease(tag)]

src/configuration.py

@@ -4,10 +4,6 @@
 
 import yaml
 
-# We want to support environment variable replacement in the configuration
-# similarly to how it is done in OGX, so we use their function directly
-from ogx.core.stack import replace_env_vars
-
 import constants
 from cache.cache import Cache
 from cache.cache_factory import CacheFactory
@@ -40,25 +36,26 @@
 from quota.quota_limiter import QuotaLimiter
 from quota.quota_limiter_factory import QuotaLimiterFactory
 from quota.token_usage_history import TokenUsageHistory
+from utils.env_vars import normalize_env_references, resolve_env_references
 
 logger = get_logger(__name__)
 
 
 def replace_env_vars_preserving_native_override(
     config_dict: dict[Any, Any],
 ) -> dict[Any, Any]:
-    """Resolve ${env.*} references in the config, except in native_override.
+    """Resolve ${env.VAR} and ${VAR} references, except in native_override.
 
     LCORE resolves environment-variable references throughout
     lightspeed-stack.yaml so typed fields receive concrete values. But
     ``ogx.config.native_override`` is raw OGX schema that OGX
     resolves itself, in memory, at its own startup. Resolving it eagerly
-    here would (a) defeat the ${env.*} pattern LCORE recommends for secrets and
+    here would (a) defeat the ${env.VAR} and ${VAR} patterns for secrets and
     (b) pull resolved secrets into the loaded Configuration model, which is
     logged at startup. So native_override is held aside, the rest of the config
-    is resolved, and the raw (unresolved) native_override is restored verbatim.
-    Synthesis reads native_override from the raw YAML, so this does not change
-    the generated run.yaml.
+    is resolved, and native_override is restored with ${VAR} normalized to
+    ${env.VAR} without resolving it. Synthesis reads native_override from the
+    raw YAML and applies the same normalization to the generated run.yaml.
 
     Parameters:
         config_dict: The parsed lightspeed-stack.yaml.
@@ -69,24 +66,24 @@ def replace_env_vars_preserving_native_override(
 
     """
     if not isinstance(config_dict, dict):
-        return replace_env_vars(config_dict)
+        return resolve_env_references(config_dict)
 
     ogx_section = config_dict.get("ogx")
     if ogx_section is None:
         ogx_section = config_dict.get("llama_stack")
     ogx_config = ogx_section.get("config") if isinstance(ogx_section, dict) else None
     if not (isinstance(ogx_config, dict) and "native_override" in ogx_config):
-        return replace_env_vars(config_dict)
+        return resolve_env_references(config_dict)
 
     raw_override = ogx_config["native_override"]
     ogx_config["native_override"] = {}  # keep secrets out of env resolution
-    resolved = replace_env_vars(config_dict)
+    resolved = resolve_env_references(config_dict)
     ogx_config["native_override"] = raw_override  # restore source dict if reused
     resolved_ogx = (resolved.get("ogx") or resolved.get("llama_stack") or {}).get(
         "config"
     )
     if isinstance(resolved_ogx, dict):
-        resolved_ogx["native_override"] = raw_override
+        resolved_ogx["native_override"] = normalize_env_references(raw_override)
     return resolved
 
 

src/ogx_configuration.py

@@ -26,11 +26,11 @@
 from urllib.parse import urljoin
 
 import yaml
-from ogx.core.stack import replace_env_vars
 from pydantic import SecretStr
 
 import constants
 from log import get_logger, setup_logging
+from utils.env_vars import normalize_env_references, resolve_env_references
 
 logger = get_logger(__name__)
 
@@ -315,10 +315,10 @@ def construct_vector_stores_section(
             output = ogx_config["registered_resources"]["vector_stores"].copy()
 
     # append new vector_stores entries, skipping duplicates
-    # Resolve ${env.VAR} patterns so comparisons work when existing entries
+    # Resolve ${env.VAR} and ${VAR} so comparisons work when existing entries
     # use environment variable references and new entries have resolved values.
     existing_store_ids = {
-        replace_env_vars(vs.get("vector_store_id", "")) for vs in output
+        resolve_env_references(vs.get("vector_store_id", "")) for vs in output
     }
     added = 0
     for brag in byok_rag:
@@ -879,8 +879,8 @@ def enrich_solr(  # pylint: disable=too-many-locals,too-many-statements,too-many
     base_url_raw = (
         str(rhokp_raw) if rhokp_raw is not None else constants.RH_SERVER_OKP_DEFAULT_URL
     )
-    # Resolve environment variables in the URL (e.g., ${env.RH_SERVER_OKP})
-    base_url = replace_env_vars(base_url_raw)
+    # Resolve ${env.RH_SERVER_OKP} and ${RH_SERVER_OKP} before joining the URL path.
+    base_url = resolve_env_references(base_url_raw)
     solr_url = urljoin(base_url, "/solr")
 
     logger.info("Enriching OGX config with OKP")
@@ -1479,6 +1479,8 @@ def synthesize_configuration(  # pylint: disable=too-many-locals
         dict[str, Any]: The synthesized OGX configuration.
 
     """
+    # Keep raw references deferred for OGX startup while canonicalizing their syntax.
+    lcs_config = normalize_env_references(lcs_config)
     unified = ogx_config_section(lcs_config).get("config")
 
     # 1-2. Select the baseline.
@@ -1712,6 +1714,9 @@ def generate_configuration(
     """
     logger.info("Reading OGX configuration from file %s", input_file)
 
+    # Normalize raw LCORE enrichment references without looking up their values.
+    config = normalize_env_references(config)
+
     with open(input_file, encoding="utf-8") as file:
         ogx_config = yaml.safe_load(file)
 

src/utils/env_vars.py

@@ -0,0 +1,69 @@
+"""Normalize environment references and resolve them using OGX semantics."""
+
+import re
+from typing import Any, Final
+
+from ogx.core.stack import replace_env_vars
+
+# Match ${VAR} and ${env.VAR}, optionally with :=default or :+value.
+# Names use OGX's uppercase/digit/underscore grammar; operands end at the first }.
+_ENV_REFERENCE_PATTERN: Final[re.Pattern[str]] = re.compile(
+    r"\$\{(?:env\.)?[A-Z0-9_]+(?::[=+][^}]*)?\}"
+)
+
+
+def canonicalize_reference(match: re.Match[str]) -> str:
+    """Preserve OGX references and prefix short references with env.
+
+    Parameters:
+        match: A supported reference found by the normalizer's regex.
+
+    Returns:
+        The reference in canonical ${env.VAR} syntax, without resolving it.
+
+    """
+    # group(0) is the complete matched reference, including ${ and }.
+    reference = match.group(0)
+    if reference.startswith("${env."):
+        return reference
+    return reference.replace("${", "${env.", 1)
+
+
+def normalize_env_references(config: Any) -> Any:
+    """Convert supported ${VAR} references to ${env.VAR} without resolving them.
+
+    Recursively visit dictionary values and list elements, rebuilding each
+    container and normalizing its strings. Keys and existing OGX references are
+    unchanged. Variable names and operators follow OGX's resolver grammar.
+
+    Parameters:
+        config: A configuration value, including nested containers and scalars.
+
+    Returns:
+        The configuration with supported short references canonicalized.
+
+    """
+    if isinstance(config, str):
+        # sub calls the canonicalizer for each match, preserving surrounding text.
+        # For ${HOST}:${PORT}, each reference gets its own canonical replacement.
+        return _ENV_REFERENCE_PATTERN.sub(canonicalize_reference, config)
+    if isinstance(config, dict):
+        return {key: normalize_env_references(value) for key, value in config.items()}
+    if isinstance(config, list):
+        return [normalize_env_references(value) for value in config]
+    return config
+
+
+def resolve_env_references(config: Any) -> Any:
+    """Normalize environment references, then delegate their resolution to OGX.
+
+    Parameters:
+        config: A configuration value, including nested containers and scalars.
+
+    Returns:
+        The recursively resolved configuration. OGX may convert substituted
+        strings to numbers, booleans, or None; other scalar values pass through.
+
+    """
+    normalized_config = normalize_env_references(config)
+    return replace_env_vars(normalized_config)

tests/conftest.py

@@ -0,0 +1,11 @@
+"""Fixtures shared by the unit and integration test suites."""
+
+import pytest
+
+
+@pytest.fixture(
+    name="reference_prefix", params=["", "env."], ids=["short", "canonical"]
+)
+def reference_prefix_fixture(request: pytest.FixtureRequest) -> str:
+    """Exercise both environment-reference spellings."""
+    return str(request.param)