feat: configure Module metadata search paths in pyproject.toml

CONFIGURATION.md

@@ -28,9 +28,16 @@ project type, with non-empty strings for `name` and
 `holoscan_sdk.minimum_required_version`. Incomplete or unrelated descriptors are
 ignored; this is not full schema validation and needs no optional dependencies.
 
-Use `HOLOSCAN_CLI_SEARCH_PATH` for custom directories such as `examples/`.
-It replaces defaults with comma-separated directories or exact `metadata.json`
-paths, relative to the root; an empty value uses conventional directories.
+To choose which folders are searched, list them in `HOLOSCAN_CLI_SEARCH_PATH`
+(comma-separated) or, for a standalone Module, in
+[`tool.holoscan.search-paths`](#pyprojecttoml-settings). The environment
+variable wins. Either list replaces the conventional directories, so it also
+excludes the ones it leaves out; `.` searches the whole project. Entries are
+directories or exact `metadata.json` paths relative to the root; an empty
+variable uses conventional directories. A Module's root `metadata.json` is
+always included. Below each searched folder, discovery skips hidden folders
+such as `.git` and `.venv`. It also skips Python and Conda environments and CMake
+build trees, except a folder with its own `metadata.json`, such as an in-source build.
 `HOLOSCAN_CLI_APP_NAME` overrides the application's directory-based selector and
 is forwarded automatically to preserve it across renamed container mounts.
 
@@ -85,6 +92,7 @@ unrelated to CLI configuration. Supported keys:
 | `forward-env` | String array / `[]` | Host variable names to forward into containers. |
 | `docker-build-args` | Token array / `[]` | Extra Docker build options. |
 | `docker-run-args` | Token array / `[]` | Extra Docker run options. |
+| `search-paths` | Relative path array / conventional directories | Folders or `metadata.json` files searched for projects; must stay within the Module root. |
 | `base-images` | Table / absent | Exact image references keyed by `x86_64` or `aarch64`. |
 
 ```toml
@@ -94,6 +102,7 @@ ctest-script = "ci/container.ctest"
 forward-env = ["IS_CI_BUILD"]
 docker-build-args = ["--build-arg", "PROJECT_FEATURE=ON"]
 docker-run-args = ["--network=host"]
+search-paths = ["operators", "examples"]
 
 [tool.holoscan.base-images]
 x86_64 = "registry.example.com/holoscan/sdk-build-x86_64:5.0.0-cuda13"

src/holoscan_cli/metadata/utils.py

@@ -55,6 +55,9 @@
 
 DEFAULT_INCLUDE_PATHS = tuple(METADATA_DIRECTORY_CONFIG.keys())
 MAX_METADATA_BYTES = 1024 * 1024
+# Entries that mark a Python or Conda environment or a CMake build tree. Such
+# folders can hold copies of project metadata, so discovery skips them.
+SKIPPED_FOLDER_MARKERS = ("pyvenv.cfg", "conda-meta", "CMakeCache.txt")
 
 
 def read_metadata(path: str | os.PathLike):
@@ -125,6 +128,26 @@ def list_normalized_languages(language, *, strict: bool = False) -> list[str]:
     return normalized or [""]
 
 
+def _walk_metadata_files(top: Path) -> Iterator[str]:
+    """Yield metadata.json files below ``top``, skipping generated folders.
+
+    Hidden folders (``.git``, ``.venv``, tool caches), Python and Conda
+    environments, and CMake build trees are skipped below ``top``. ``top`` and
+    folders with their own metadata.json, such as in-source builds, are searched.
+    """
+    for root, dirs, files in os.walk(top):
+        if (
+            root != os.fspath(top)
+            and "metadata.json" not in files
+            and not {*files, *dirs}.isdisjoint(SKIPPED_FOLDER_MARKERS)
+        ):
+            dirs.clear()
+            continue
+        dirs[:] = [name for name in dirs if not name.startswith(".")]
+        if "metadata.json" in files:
+            yield os.path.join(root, "metadata.json")
+
+
 def iter_metadata_paths(
     repo_paths: Sequence[str | os.PathLike],
     *,
@@ -143,16 +166,14 @@ def _matches_segment(path: str, patterns: Sequence[str]) -> bool:
             ("/" + pattern.replace(os.sep, "/").strip("/") + "/") in norm for pattern in patterns
         )
 
+    # Overlapping search paths, such as "." and "applications", list each file once.
+    seen: set[str] = set()
     for repo_path in repo_paths:
         path = Path(repo_path)
         if path.is_file():
             candidates = [str(path)] if path.name == "metadata.json" else []
         else:
-            candidates = [
-                os.path.join(root, "metadata.json")
-                for root, _, files in os.walk(path)
-                if "metadata.json" in files
-            ]
+            candidates = _walk_metadata_files(path)
 
         for file_path in candidates:
             if excludes and _matches_segment(file_path, excludes):
@@ -168,4 +189,9 @@ def _matches_segment(path: str, patterns: Sequence[str]) -> bool:
             if ignore_patterns and _matches_segment(file_path, ignore_patterns):
                 continue
 
+            # Resolve symlinks so aliased search folders list a file once.
+            resolved_path = os.path.realpath(file_path)
+            if resolved_path in seen:
+                continue
+            seen.add(resolved_path)
             yield file_path

src/holoscan_cli/project_context.py

@@ -89,6 +89,7 @@ class ProjectContext:
     docker_build_args: Optional[str] = None
     docker_run_args: Optional[str] = None
     forward_env: tuple[str, ...] = ()
+    search_paths: tuple[str, ...] = ()
     warnings: tuple[str, ...] = ()
     container_prefix: Optional[str] = None
     workspace_name: Optional[str] = None
@@ -113,7 +114,9 @@ def profile_environment(self) -> dict[str, str]:
             "HOLOSCAN_CLI_ROOT": str(self.root),
             "HOLOSCAN_CLI_BUILD_PARENT_DIR": str(self.root / "build"),
             "HOLOSCAN_CLI_DATA_DIR": str(self.root / "data"),
-            "HOLOSCAN_CLI_SEARCH_PATH": ",".join((MODULE_METADATA_FILENAME, *SEARCH_DIRS)),
+            "HOLOSCAN_CLI_SEARCH_PATH": ",".join(
+                (MODULE_METADATA_FILENAME, *(self.search_paths or SEARCH_DIRS))
+            ),
         }
         optional_values = {
             "HOLOSCAN_CLI_REPO_PREFIX": self.repo_prefix,
@@ -238,6 +241,7 @@ def _read_holoscan_project_config(root: Path) -> tuple[Optional[Path], dict]:
             "docker-run-args",
             "forward-env",
             "repo-prefix",
+            "search-paths",
             "workspace-name",
         },
         source_path=config_path,
@@ -307,6 +311,35 @@ def _resolve_project_profile(
             source_path=config_source,
         )
 
+    search_paths = config.get("search-paths")
+    if search_paths is not None:
+        # Entries are joined into the comma-separated HOLOSCAN_CLI_SEARCH_PATH.
+        if (
+            not isinstance(search_paths, list)
+            or not search_paths
+            or any(isinstance(value, str) and "," in value for value in search_paths)
+        ):
+            raise ProjectContextError(
+                f"{config_source}: tool.holoscan.search-paths must be a non-empty array of "
+                "relative paths without commas."
+            )
+        resolved["search_paths"] = tuple(
+            relative_project_path(
+                value,
+                path="tool.holoscan.search-paths",
+                source_path=config_source,
+            )
+            for value in search_paths
+        )
+        # A symlinked entry must not lead discovery outside the Module.
+        real_root = root.resolve()
+        for value in resolved["search_paths"]:
+            if not (root / value).resolve().is_relative_to(real_root):
+                raise ProjectContextError(
+                    f"{config_source}: tool.holoscan.search-paths must stay within the "
+                    f"project and be relative: {value}"
+                )
+
     for key, field in (
         ("docker-build-args", "docker_build_args"),
         ("docker-run-args", "docker_run_args"),
@@ -521,6 +554,7 @@ def _build_module_context(
         docker_build_args=profile.get("docker_build_args"),
         docker_run_args=profile.get("docker_run_args"),
         forward_env=profile.get("forward_env", ()),
+        search_paths=profile.get("search_paths", ()),
         warnings=(*warnings, *profile_warnings),
     )
 

src/holoscan_cli/templates/module/{{cookiecutter.module_repo_name}}/DEVELOPER.md

@@ -166,9 +166,12 @@ update before publishing:
 
 The supported keys are `tool.holoscan.cuda`, `tool.holoscan.ctest-script`,
 `tool.holoscan.docker-build-args`, `tool.holoscan.docker-run-args`,
-`tool.holoscan.forward-env`, and `tool.holoscan.base-images.{x86_64,aarch64}`. Docker argument
-arrays contain one command token per element. Do not store credentials or machine paths here;
-use `HOLOSCAN_SDK_ROOT` or `--local-sdk-root` for a machine-specific SDK.
+`tool.holoscan.forward-env`, `tool.holoscan.search-paths`, and
+`tool.holoscan.base-images.{x86_64,aarch64}`. Docker argument arrays contain one command token
+per element. `search-paths` lists the folders searched for projects, replacing the defaults
+(`applications`, `operators`, `pkg`, and other conventional folders); for example,
+`search-paths = ["operators", "examples"]`. Do not store credentials or machine paths here; use `HOLOSCAN_SDK_ROOT` or
+`--local-sdk-root` for a machine-specific SDK.
 
 Build a wheel:
 

src/holoscan_cli/utils/holohub.py

@@ -34,9 +34,11 @@
 from typing import Mapping, Optional, Tuple
 
 from holoscan_cli.project_context import (
+    MODULE_METADATA_FILENAME,
     SEARCH_DIRS,
     activated_environment_source,
     discover_project_context,
+    get_active_project_context,
 )
 from holoscan_cli.utils.io import format_cmd, info, run_info_command, warn
 from holoscan_cli.utils.project import CMAKE_BUILD_TYPES, resolve_build_type
@@ -109,7 +111,11 @@ def get_component_search_paths(base_dir: Optional[Path] = None) -> tuple[Path, .
     """Return metadata search paths honoring HOLOSCAN_CLI_SEARCH_PATH overrides."""
     base_path = base_dir or get_holohub_root()
     tokens = os.environ.get("HOLOSCAN_CLI_SEARCH_PATH", "").split(",")
-    paths = [token.strip() for token in tokens if token.strip()] or SEARCH_DIRS
+    paths = [token.strip() for token in tokens if token.strip()] or list(SEARCH_DIRS)
+    context = get_active_project_context()
+    if context is not None and context.is_module and context.root == base_path:
+        # A custom search path must not hide the Module itself.
+        paths.insert(0, MODULE_METADATA_FILENAME)
     return tuple(
         (Path(token) if Path(token).is_absolute() else base_path / token) for token in paths
     )

tests/unit/test_gather_metadata.py

@@ -40,3 +40,52 @@ def test_module_uses_declared_name_instead_of_mount_directory(tmp_path):
         projects = gather_metadata([str(mounted_root)])
 
         assert projects[0]["project_name"] == expected_name
+
+
+def test_search_skips_generated_folders_and_lists_each_file_once(tmp_path):
+    root = tmp_path / "module"
+    application = json.dumps({"application": {"name": "ex_app"}})
+    for folder in (
+        "examples/ex_app",
+        ".venv/lib/placeholder",  # hidden folder
+        "venv/lib/placeholder",  # virtual environment
+        "conda/lib/placeholder",  # Conda environment
+        "build/ex_app/examples/ex_app",  # CMake build tree
+        "examples/in_source_app",  # in-source CMake build
+    ):
+        (root / folder).mkdir(parents=True)
+        (root / folder / "metadata.json").write_text(application, encoding="utf-8")
+    (root / "metadata.json").write_text(json.dumps({"module": {"name": "m"}}), encoding="utf-8")
+    (root / "venv/pyvenv.cfg").touch()
+    (root / "conda/conda-meta").mkdir()
+    (root / "build/ex_app/CMakeCache.txt").touch()
+    (root / "examples/in_source_app/CMakeCache.txt").touch()
+
+    projects = gather_metadata([str(root / "metadata.json"), str(root), str(root / "examples")])
+
+    assert sorted(project["source_folder"] for project in projects) == [
+        str(root),
+        str(root / "examples/ex_app"),
+        str(root / "examples/in_source_app"),
+    ]
+    # A folder named directly in the search path is always searched.
+    assert len(gather_metadata([str(root / "build/ex_app")])) == 1
+
+
+def test_search_lists_a_file_reached_through_a_symlinked_folder_once(tmp_path):
+    root = tmp_path / "module"
+    (root / "applications/ex_app").mkdir(parents=True)
+    (root / "applications/ex_app/metadata.json").write_text(
+        json.dumps({"application": {"name": "ex_app"}}), encoding="utf-8"
+    )
+    (root / "examples").symlink_to(root / "applications", target_is_directory=True)
+
+    assert len(gather_metadata([str(root / "examples")])) == 1
+    projects = gather_metadata([str(root / "applications"), str(root / "examples")])
+
+    assert [project["source_folder"] for project in projects] == [str(root / "applications/ex_app")]
+    # An excluded path doesn't hide the same file reached through another path.
+    projects = gather_metadata(
+        [str(root / "applications"), str(root / "examples")], exclude_paths=["applications"]
+    )
+    assert [project["source_folder"] for project in projects] == [str(root / "examples/ex_app")]

tests/unit/test_project_context.py

@@ -139,6 +139,7 @@ def test_module_pyproject_resolves_project_contract_and_nearby_sdk(
 forward-env = ["CI"]
 docker-build-args = ["--build-arg", "PROJECT_FEATURE=ON"]
 docker-run-args = ["--network=host"]
+search-paths = ["examples", "./tools/"]
 
 [tool.holoscan.base-images]
 x86_64 = "example.test/holoscan:fixed"
@@ -157,6 +158,8 @@ def test_module_pyproject_resolves_project_contract_and_nearby_sdk(
     assert context.docker_build_args == "--build-arg PROJECT_FEATURE=ON"
     assert context.docker_run_args == "--network=host"
     assert context.forward_env == ("CI",)
+    assert context.search_paths == ("examples", "tools")
+    assert profile["HOLOSCAN_CLI_SEARCH_PATH"] == "metadata.json,examples,tools"
     assert profile["HOLOSCAN_CLI_DEFAULT_CUDA_VERSION"] == "13"
     assert profile["HOLOSCAN_CLI_CTEST_SCRIPT"] == "ci/container.ctest"
     assert profile["HOLOSCAN_CLI_REPO_PREFIX"] == "sensor_repo"
@@ -208,6 +211,10 @@ def test_module_environment_resolves_4x_sdk_build_tree(tmp_path, monkeypatch, ma
         ("[tool.holoscan]\nctest-script = '../container.ctest'\n", "ctest-script"),
         ("[tool.holoscan]\nforward-env = ['HOME']\n", "forward-env"),
         ("[tool.holoscan]\ndocker-run-args = ['']\n", "docker-run-args"),
+        ("[tool.holoscan]\nsearch-paths = 'examples'\n", "search-paths must"),
+        ("[tool.holoscan]\nsearch-paths = []\n", "search-paths must"),
+        ("[tool.holoscan]\nsearch-paths = ['../examples']\n", "search-paths must"),
+        ("[tool.holoscan]\nsearch-paths = ['a,b']\n", "search-paths must"),
         ("[tool.holoscan]\nbase-images = 'x86_64'\n", "base-images must be a table"),
         (
             "[tool.holoscan.base-images]\n"
@@ -224,6 +231,15 @@ def test_module_pyproject_rejects_unsafe_or_unknown_values(tmp_path, pyproject,
         discover_project_context(cwd=root, environ={})
 
 
+def test_module_search_paths_reject_a_symlink_outside_the_module(tmp_path):
+    root = _module(tmp_path / "module", pyproject="[tool.holoscan]\nsearch-paths = ['examples']\n")
+    (tmp_path / "outside").mkdir()
+    (root / "examples").symlink_to(tmp_path / "outside", target_is_directory=True)
+
+    with pytest.raises(ProjectContextError, match="search-paths must stay within"):
+        discover_project_context(cwd=root, environ={})
+
+
 def test_lightweight_import_does_not_load_project_cli(tmp_path):
     script = """
 import sys
@@ -271,3 +287,40 @@ def test_dispatch_activates_module_without_a_runtime_version_gate(tmp_path):
     projects = json.loads(result.stdout)["projects"]
     module = next(project for project in projects if project["project_type"] == "module")
     assert module["source_folder"] == str(root)
+
+
+def test_list_uses_search_paths_and_skips_generated_folders(tmp_path):
+    root = _module(
+        tmp_path / "module",
+        pyproject='[tool.holoscan]\nsearch-paths = ["examples"]\n',
+    )
+    application = json.dumps({"application": {"name": "ex_app"}})
+    for folder in (
+        "examples/ex_app",
+        "operators/ex_op",
+        ".venv/lib/placeholder",
+        "build/ex_app/examples/ex_app",
+    ):
+        (root / folder).mkdir(parents=True)
+        (root / folder / "metadata.json").write_text(application, encoding="utf-8")
+    (root / "build/ex_app/CMakeCache.txt").touch()
+
+    def list_projects(**environment: str) -> list[tuple[str, str]]:
+        result = subprocess.run(
+            [sys.executable, "-m", "holoscan_cli", "--project-root", str(root), "list", "--json"],
+            cwd=tmp_path,
+            env={**_subprocess_env(), **environment},
+            capture_output=True,
+            text=True,
+        )
+        assert result.returncode == 0, result.stderr
+        projects = json.loads(result.stdout)["projects"]
+        return sorted((project["project_type"], project["source_folder"]) for project in projects)
+
+    module = ("module", str(root))
+    example = ("application", str(root / "examples/ex_app"))
+    operator = ("application", str(root / "operators/ex_op"))
+    # search-paths replaces the default folders; the Module itself is always listed.
+    assert list_projects() == [example, module]
+    assert list_projects(HOLOSCAN_CLI_SEARCH_PATH="operators") == [operator, module]
+    assert list_projects(HOLOSCAN_CLI_SEARCH_PATH=".") == [example, operator, module]