diff --git a/CONFIGURATION.md b/CONFIGURATION.md index ccc1c881..f9373fbd 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -28,9 +28,16 @@ project type, with non-empty strings for `name` and `holoscan_sdk.minimum_required_version`. Incomplete or unrelated descriptors are ignored; this is not full schema validation and needs no optional dependencies. -Use `HOLOSCAN_CLI_SEARCH_PATH` for custom directories such as `examples/`. -It replaces defaults with comma-separated directories or exact `metadata.json` -paths, relative to the root; an empty value uses conventional directories. +To choose which folders are searched, list them in `HOLOSCAN_CLI_SEARCH_PATH` +(comma-separated) or, for a standalone Module, in +[`tool.holoscan.search-paths`](#pyprojecttoml-settings). The environment +variable wins. Either list replaces the conventional directories, so it also +excludes the ones it leaves out; `.` searches the whole project. Entries are +directories or exact `metadata.json` paths relative to the root; an empty +variable uses conventional directories. A Module's root `metadata.json` is +always included. Below each searched folder, discovery skips hidden folders +such as `.git` and `.venv`. It also skips Python and Conda environments and CMake +build trees, except a folder with its own `metadata.json`, such as an in-source build. `HOLOSCAN_CLI_APP_NAME` overrides the application's directory-based selector and is forwarded automatically to preserve it across renamed container mounts. @@ -85,6 +92,7 @@ unrelated to CLI configuration. Supported keys: | `forward-env` | String array / `[]` | Host variable names to forward into containers. | | `docker-build-args` | Token array / `[]` | Extra Docker build options. | | `docker-run-args` | Token array / `[]` | Extra Docker run options. | +| `search-paths` | Relative path array / conventional directories | Folders or `metadata.json` files searched for projects; must stay within the Module root. | | `base-images` | Table / absent | Exact image references keyed by `x86_64` or `aarch64`. | ```toml @@ -94,6 +102,7 @@ ctest-script = "ci/container.ctest" forward-env = ["IS_CI_BUILD"] docker-build-args = ["--build-arg", "PROJECT_FEATURE=ON"] docker-run-args = ["--network=host"] +search-paths = ["operators", "examples"] [tool.holoscan.base-images] x86_64 = "registry.example.com/holoscan/sdk-build-x86_64:5.0.0-cuda13" diff --git a/src/holoscan_cli/metadata/utils.py b/src/holoscan_cli/metadata/utils.py index aa77ed73..db3115a7 100644 --- a/src/holoscan_cli/metadata/utils.py +++ b/src/holoscan_cli/metadata/utils.py @@ -55,6 +55,9 @@ DEFAULT_INCLUDE_PATHS = tuple(METADATA_DIRECTORY_CONFIG.keys()) MAX_METADATA_BYTES = 1024 * 1024 +# Entries that mark a Python or Conda environment or a CMake build tree. Such +# folders can hold copies of project metadata, so discovery skips them. +SKIPPED_FOLDER_MARKERS = ("pyvenv.cfg", "conda-meta", "CMakeCache.txt") def read_metadata(path: str | os.PathLike): @@ -125,6 +128,26 @@ def list_normalized_languages(language, *, strict: bool = False) -> list[str]: return normalized or [""] +def _walk_metadata_files(top: Path) -> Iterator[str]: + """Yield metadata.json files below ``top``, skipping generated folders. + + Hidden folders (``.git``, ``.venv``, tool caches), Python and Conda + environments, and CMake build trees are skipped below ``top``. ``top`` and + folders with their own metadata.json, such as in-source builds, are searched. + """ + for root, dirs, files in os.walk(top): + if ( + root != os.fspath(top) + and "metadata.json" not in files + and not {*files, *dirs}.isdisjoint(SKIPPED_FOLDER_MARKERS) + ): + dirs.clear() + continue + dirs[:] = [name for name in dirs if not name.startswith(".")] + if "metadata.json" in files: + yield os.path.join(root, "metadata.json") + + def iter_metadata_paths( repo_paths: Sequence[str | os.PathLike], *, @@ -143,16 +166,14 @@ def _matches_segment(path: str, patterns: Sequence[str]) -> bool: ("/" + pattern.replace(os.sep, "/").strip("/") + "/") in norm for pattern in patterns ) + # Overlapping search paths, such as "." and "applications", list each file once. + seen: set[str] = set() for repo_path in repo_paths: path = Path(repo_path) if path.is_file(): candidates = [str(path)] if path.name == "metadata.json" else [] else: - candidates = [ - os.path.join(root, "metadata.json") - for root, _, files in os.walk(path) - if "metadata.json" in files - ] + candidates = _walk_metadata_files(path) for file_path in candidates: if excludes and _matches_segment(file_path, excludes): @@ -168,4 +189,9 @@ def _matches_segment(path: str, patterns: Sequence[str]) -> bool: if ignore_patterns and _matches_segment(file_path, ignore_patterns): continue + # Resolve symlinks so aliased search folders list a file once. + resolved_path = os.path.realpath(file_path) + if resolved_path in seen: + continue + seen.add(resolved_path) yield file_path diff --git a/src/holoscan_cli/project_context.py b/src/holoscan_cli/project_context.py index 3a6c809a..07cfd0a3 100644 --- a/src/holoscan_cli/project_context.py +++ b/src/holoscan_cli/project_context.py @@ -89,6 +89,7 @@ class ProjectContext: docker_build_args: Optional[str] = None docker_run_args: Optional[str] = None forward_env: tuple[str, ...] = () + search_paths: tuple[str, ...] = () warnings: tuple[str, ...] = () container_prefix: Optional[str] = None workspace_name: Optional[str] = None @@ -113,7 +114,9 @@ def profile_environment(self) -> dict[str, str]: "HOLOSCAN_CLI_ROOT": str(self.root), "HOLOSCAN_CLI_BUILD_PARENT_DIR": str(self.root / "build"), "HOLOSCAN_CLI_DATA_DIR": str(self.root / "data"), - "HOLOSCAN_CLI_SEARCH_PATH": ",".join((MODULE_METADATA_FILENAME, *SEARCH_DIRS)), + "HOLOSCAN_CLI_SEARCH_PATH": ",".join( + (MODULE_METADATA_FILENAME, *(self.search_paths or SEARCH_DIRS)) + ), } optional_values = { "HOLOSCAN_CLI_REPO_PREFIX": self.repo_prefix, @@ -238,6 +241,7 @@ def _read_holoscan_project_config(root: Path) -> tuple[Optional[Path], dict]: "docker-run-args", "forward-env", "repo-prefix", + "search-paths", "workspace-name", }, source_path=config_path, @@ -307,6 +311,35 @@ def _resolve_project_profile( source_path=config_source, ) + search_paths = config.get("search-paths") + if search_paths is not None: + # Entries are joined into the comma-separated HOLOSCAN_CLI_SEARCH_PATH. + if ( + not isinstance(search_paths, list) + or not search_paths + or any(isinstance(value, str) and "," in value for value in search_paths) + ): + raise ProjectContextError( + f"{config_source}: tool.holoscan.search-paths must be a non-empty array of " + "relative paths without commas." + ) + resolved["search_paths"] = tuple( + relative_project_path( + value, + path="tool.holoscan.search-paths", + source_path=config_source, + ) + for value in search_paths + ) + # A symlinked entry must not lead discovery outside the Module. + real_root = root.resolve() + for value in resolved["search_paths"]: + if not (root / value).resolve().is_relative_to(real_root): + raise ProjectContextError( + f"{config_source}: tool.holoscan.search-paths must stay within the " + f"project and be relative: {value}" + ) + for key, field in ( ("docker-build-args", "docker_build_args"), ("docker-run-args", "docker_run_args"), @@ -521,6 +554,7 @@ def _build_module_context( docker_build_args=profile.get("docker_build_args"), docker_run_args=profile.get("docker_run_args"), forward_env=profile.get("forward_env", ()), + search_paths=profile.get("search_paths", ()), warnings=(*warnings, *profile_warnings), ) diff --git a/src/holoscan_cli/templates/module/{{cookiecutter.module_repo_name}}/DEVELOPER.md b/src/holoscan_cli/templates/module/{{cookiecutter.module_repo_name}}/DEVELOPER.md index 5c105397..eb3c4f84 100644 --- a/src/holoscan_cli/templates/module/{{cookiecutter.module_repo_name}}/DEVELOPER.md +++ b/src/holoscan_cli/templates/module/{{cookiecutter.module_repo_name}}/DEVELOPER.md @@ -166,9 +166,12 @@ update before publishing: The supported keys are `tool.holoscan.cuda`, `tool.holoscan.ctest-script`, `tool.holoscan.docker-build-args`, `tool.holoscan.docker-run-args`, -`tool.holoscan.forward-env`, and `tool.holoscan.base-images.{x86_64,aarch64}`. Docker argument -arrays contain one command token per element. Do not store credentials or machine paths here; -use `HOLOSCAN_SDK_ROOT` or `--local-sdk-root` for a machine-specific SDK. +`tool.holoscan.forward-env`, `tool.holoscan.search-paths`, and +`tool.holoscan.base-images.{x86_64,aarch64}`. Docker argument arrays contain one command token +per element. `search-paths` lists the folders searched for projects, replacing the defaults +(`applications`, `operators`, `pkg`, and other conventional folders); for example, +`search-paths = ["operators", "examples"]`. Do not store credentials or machine paths here; use `HOLOSCAN_SDK_ROOT` or +`--local-sdk-root` for a machine-specific SDK. Build a wheel: diff --git a/src/holoscan_cli/utils/holohub.py b/src/holoscan_cli/utils/holohub.py index 7b2de0b4..fde56aaf 100644 --- a/src/holoscan_cli/utils/holohub.py +++ b/src/holoscan_cli/utils/holohub.py @@ -34,9 +34,11 @@ from typing import Mapping, Optional, Tuple from holoscan_cli.project_context import ( + MODULE_METADATA_FILENAME, SEARCH_DIRS, activated_environment_source, discover_project_context, + get_active_project_context, ) from holoscan_cli.utils.io import format_cmd, info, run_info_command, warn from holoscan_cli.utils.project import CMAKE_BUILD_TYPES, resolve_build_type @@ -109,7 +111,11 @@ def get_component_search_paths(base_dir: Optional[Path] = None) -> tuple[Path, . """Return metadata search paths honoring HOLOSCAN_CLI_SEARCH_PATH overrides.""" base_path = base_dir or get_holohub_root() tokens = os.environ.get("HOLOSCAN_CLI_SEARCH_PATH", "").split(",") - paths = [token.strip() for token in tokens if token.strip()] or SEARCH_DIRS + paths = [token.strip() for token in tokens if token.strip()] or list(SEARCH_DIRS) + context = get_active_project_context() + if context is not None and context.is_module and context.root == base_path: + # A custom search path must not hide the Module itself. + paths.insert(0, MODULE_METADATA_FILENAME) return tuple( (Path(token) if Path(token).is_absolute() else base_path / token) for token in paths ) diff --git a/tests/unit/test_gather_metadata.py b/tests/unit/test_gather_metadata.py index 44fb7202..cd39ac0c 100644 --- a/tests/unit/test_gather_metadata.py +++ b/tests/unit/test_gather_metadata.py @@ -40,3 +40,52 @@ def test_module_uses_declared_name_instead_of_mount_directory(tmp_path): projects = gather_metadata([str(mounted_root)]) assert projects[0]["project_name"] == expected_name + + +def test_search_skips_generated_folders_and_lists_each_file_once(tmp_path): + root = tmp_path / "module" + application = json.dumps({"application": {"name": "ex_app"}}) + for folder in ( + "examples/ex_app", + ".venv/lib/placeholder", # hidden folder + "venv/lib/placeholder", # virtual environment + "conda/lib/placeholder", # Conda environment + "build/ex_app/examples/ex_app", # CMake build tree + "examples/in_source_app", # in-source CMake build + ): + (root / folder).mkdir(parents=True) + (root / folder / "metadata.json").write_text(application, encoding="utf-8") + (root / "metadata.json").write_text(json.dumps({"module": {"name": "m"}}), encoding="utf-8") + (root / "venv/pyvenv.cfg").touch() + (root / "conda/conda-meta").mkdir() + (root / "build/ex_app/CMakeCache.txt").touch() + (root / "examples/in_source_app/CMakeCache.txt").touch() + + projects = gather_metadata([str(root / "metadata.json"), str(root), str(root / "examples")]) + + assert sorted(project["source_folder"] for project in projects) == [ + str(root), + str(root / "examples/ex_app"), + str(root / "examples/in_source_app"), + ] + # A folder named directly in the search path is always searched. + assert len(gather_metadata([str(root / "build/ex_app")])) == 1 + + +def test_search_lists_a_file_reached_through_a_symlinked_folder_once(tmp_path): + root = tmp_path / "module" + (root / "applications/ex_app").mkdir(parents=True) + (root / "applications/ex_app/metadata.json").write_text( + json.dumps({"application": {"name": "ex_app"}}), encoding="utf-8" + ) + (root / "examples").symlink_to(root / "applications", target_is_directory=True) + + assert len(gather_metadata([str(root / "examples")])) == 1 + projects = gather_metadata([str(root / "applications"), str(root / "examples")]) + + assert [project["source_folder"] for project in projects] == [str(root / "applications/ex_app")] + # An excluded path doesn't hide the same file reached through another path. + projects = gather_metadata( + [str(root / "applications"), str(root / "examples")], exclude_paths=["applications"] + ) + assert [project["source_folder"] for project in projects] == [str(root / "examples/ex_app")] diff --git a/tests/unit/test_project_context.py b/tests/unit/test_project_context.py index a590e91b..4fb048b1 100644 --- a/tests/unit/test_project_context.py +++ b/tests/unit/test_project_context.py @@ -139,6 +139,7 @@ def test_module_pyproject_resolves_project_contract_and_nearby_sdk( forward-env = ["CI"] docker-build-args = ["--build-arg", "PROJECT_FEATURE=ON"] docker-run-args = ["--network=host"] +search-paths = ["examples", "./tools/"] [tool.holoscan.base-images] x86_64 = "example.test/holoscan:fixed" @@ -157,6 +158,8 @@ def test_module_pyproject_resolves_project_contract_and_nearby_sdk( assert context.docker_build_args == "--build-arg PROJECT_FEATURE=ON" assert context.docker_run_args == "--network=host" assert context.forward_env == ("CI",) + assert context.search_paths == ("examples", "tools") + assert profile["HOLOSCAN_CLI_SEARCH_PATH"] == "metadata.json,examples,tools" assert profile["HOLOSCAN_CLI_DEFAULT_CUDA_VERSION"] == "13" assert profile["HOLOSCAN_CLI_CTEST_SCRIPT"] == "ci/container.ctest" assert profile["HOLOSCAN_CLI_REPO_PREFIX"] == "sensor_repo" @@ -208,6 +211,10 @@ def test_module_environment_resolves_4x_sdk_build_tree(tmp_path, monkeypatch, ma ("[tool.holoscan]\nctest-script = '../container.ctest'\n", "ctest-script"), ("[tool.holoscan]\nforward-env = ['HOME']\n", "forward-env"), ("[tool.holoscan]\ndocker-run-args = ['']\n", "docker-run-args"), + ("[tool.holoscan]\nsearch-paths = 'examples'\n", "search-paths must"), + ("[tool.holoscan]\nsearch-paths = []\n", "search-paths must"), + ("[tool.holoscan]\nsearch-paths = ['../examples']\n", "search-paths must"), + ("[tool.holoscan]\nsearch-paths = ['a,b']\n", "search-paths must"), ("[tool.holoscan]\nbase-images = 'x86_64'\n", "base-images must be a table"), ( "[tool.holoscan.base-images]\n" @@ -224,6 +231,15 @@ def test_module_pyproject_rejects_unsafe_or_unknown_values(tmp_path, pyproject, discover_project_context(cwd=root, environ={}) +def test_module_search_paths_reject_a_symlink_outside_the_module(tmp_path): + root = _module(tmp_path / "module", pyproject="[tool.holoscan]\nsearch-paths = ['examples']\n") + (tmp_path / "outside").mkdir() + (root / "examples").symlink_to(tmp_path / "outside", target_is_directory=True) + + with pytest.raises(ProjectContextError, match="search-paths must stay within"): + discover_project_context(cwd=root, environ={}) + + def test_lightweight_import_does_not_load_project_cli(tmp_path): script = """ import sys @@ -271,3 +287,40 @@ def test_dispatch_activates_module_without_a_runtime_version_gate(tmp_path): projects = json.loads(result.stdout)["projects"] module = next(project for project in projects if project["project_type"] == "module") assert module["source_folder"] == str(root) + + +def test_list_uses_search_paths_and_skips_generated_folders(tmp_path): + root = _module( + tmp_path / "module", + pyproject='[tool.holoscan]\nsearch-paths = ["examples"]\n', + ) + application = json.dumps({"application": {"name": "ex_app"}}) + for folder in ( + "examples/ex_app", + "operators/ex_op", + ".venv/lib/placeholder", + "build/ex_app/examples/ex_app", + ): + (root / folder).mkdir(parents=True) + (root / folder / "metadata.json").write_text(application, encoding="utf-8") + (root / "build/ex_app/CMakeCache.txt").touch() + + def list_projects(**environment: str) -> list[tuple[str, str]]: + result = subprocess.run( + [sys.executable, "-m", "holoscan_cli", "--project-root", str(root), "list", "--json"], + cwd=tmp_path, + env={**_subprocess_env(), **environment}, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + projects = json.loads(result.stdout)["projects"] + return sorted((project["project_type"], project["source_folder"]) for project in projects) + + module = ("module", str(root)) + example = ("application", str(root / "examples/ex_app")) + operator = ("application", str(root / "operators/ex_op")) + # search-paths replaces the default folders; the Module itself is always listed. + assert list_projects() == [example, module] + assert list_projects(HOLOSCAN_CLI_SEARCH_PATH="operators") == [operator, module] + assert list_projects(HOLOSCAN_CLI_SEARCH_PATH=".") == [example, operator, module]