feat(files): serve a fileset straight from a GitHub repository [ASTD-519]

docs/set-up/config-reference.mdx

@@ -245,8 +245,8 @@ files:
     path: /data/files_storage
     # How many bytes to buffer before flushing to disk | default: 16777216
     write_buffer_size: 16777216
-  # Comma-separated list of external hosts the Files service is allowed to access. | default: 'https://api.ngc.nvidia.com,https://huggingface.co'
-  allowed_external_hosts: https://api.ngc.nvidia.com,https://huggingface.co
+  # Comma-separated list of external hosts the Files service is allowed to access. | default: 'https://api.ngc.nvidia.com,https://huggingface.co,https://api.github.com'
+  allowed_external_hosts: https://api.ngc.nvidia.com,https://huggingface.co,https://api.github.com
   # Allow users to explicitly create filesets with local storage config. Security-sensitive: enable only in trusted deployments. | default: False
   allow_user_local_storage: false
   # TTL for file locks in seconds (default 5 minutes) | default: 300

packages/nemo_platform_plugin/src/nemo_platform_plugin/files/storage_config.py

@@ -27,6 +27,7 @@ class StorageConfigType(StrEnum):
     NGC = "ngc"
     HUGGINGFACE = "huggingface"
     S3 = "s3"
+    GITHUB = "github"
     # AZURE_BLOB = "azure_blob"
     # GCS = "gcs"
     # HTTP = "http"
@@ -146,6 +147,43 @@ def get_secret_references(self) -> dict[str, SecretRef]:
         return {"token": self.token_secret} if self.token_secret else {}
 
 
+class GithubStorageConfig(BaseStorageConfig):
+    type: Literal[StorageConfigType.GITHUB] = StorageConfigType.GITHUB
+    owner: str = Field(description="GitHub repository owner (user or organization)")
+    repo: str = Field(description="GitHub repository name")
+    revision: str = Field(
+        default="HEAD",
+        description="Branch, tag, or commit SHA. 'HEAD' resolves to the repository's default branch.",
+    )
+    original_revision: str | None = Field(
+        default=None,
+        description="The original revision requested by the user before resolution (e.g., 'main'). "
+        "The 'revision' field contains the resolved commit SHA.",
+    )
+    path: str = Field(
+        default="",
+        description="Optional directory within the repository. All paths are relative to it.",
+    )
+
+    token_secret: SecretRef | None = Field(
+        default=None,
+        description="GitHub personal access token secret name, required for private repositories",
+    )
+
+    api_base_url: str = Field(
+        default="https://api.github.com",
+        description="GitHub API base URL. Use for GitHub Enterprise instances.",
+    )
+
+    @field_validator("path")
+    @classmethod
+    def strip_path_slashes(cls, v: str) -> str:
+        return v.strip("/")
+
+    def get_secret_references(self) -> dict[str, SecretRef]:
+        return {"token": self.token_secret} if self.token_secret else {}
+
+
 class NGCStorageConfig(BaseStorageConfig):
     type: Literal[StorageConfigType.NGC] = StorageConfigType.NGC
     org: str = Field(description="NGC organization name")
@@ -252,6 +290,6 @@ def copy_config(self, path: str) -> Self:
         return self.model_copy(deep=True, update={"prefix": new_prefix})
 
 
-StorageConfig = LocalStorageConfig | NGCStorageConfig | HuggingfaceStorageConfig | S3StorageConfig
+StorageConfig = LocalStorageConfig | NGCStorageConfig | HuggingfaceStorageConfig | S3StorageConfig | GithubStorageConfig
 
 StorageConfigField = Annotated[StorageConfig, Field(discriminator="type")]

packages/nmp_common/src/nmp/common/files/storage_config.py

@@ -10,6 +10,7 @@
 
 from nemo_platform_plugin.files.storage_config import DEFAULT_READ_CHUNK_SIZE as DEFAULT_READ_CHUNK_SIZE
 from nemo_platform_plugin.files.storage_config import BaseStorageConfig as BaseStorageConfig
+from nemo_platform_plugin.files.storage_config import GithubStorageConfig as GithubStorageConfig
 from nemo_platform_plugin.files.storage_config import HuggingfaceStorageConfig as HuggingfaceStorageConfig
 from nemo_platform_plugin.files.storage_config import LocalStorageConfig as LocalStorageConfig
 from nemo_platform_plugin.files.storage_config import NGCStorageConfig as NGCStorageConfig

services/core/files/src/nmp/core/files/app/backends/factory.py

@@ -14,6 +14,7 @@
 import logging
 
 from nmp.common.files.storage_config import (
+    GithubStorageConfig,
     HuggingfaceStorageConfig,
     LocalStorageConfig,
     NGCStorageConfig,
@@ -49,5 +50,9 @@ def storage_impl_factory(
             from nmp.core.files.app.backends.s3 import S3StorageImpl
 
             return S3StorageImpl(config, secrets)
+        case GithubStorageConfig():
+            from nmp.core.files.app.backends.github import GithubStorageImpl
+
+            return GithubStorageImpl(config, secrets)
         case _:
             raise TypeError(f"Unsupported storage config type: {type(config).__name__}")